Compare commits

...

23 Commits

Author SHA1 Message Date
bel
7cd5e5467f Merge branch 'master' of http://gogs.scratch.com:59515/bel/truckstop 2022-01-27 16:35:08 -07:00
Bel LaPointe
d456de12e8 logs but no changes 2022-01-27 16:34:53 -07:00
Bel LaPointe
c56acbc120 Merge branch 'master' of http://gogs.scratch.com:59515/bel/truckstop 2022-01-27 16:24:21 -07:00
Bel LaPointe
ae4c5bd886 backward compatible from ID to uid for sent_jobs 2022-01-27 16:22:58 -07:00
bel
4f27976fbb log because i think i deploy bad 2022-01-27 16:05:17 -07:00
bel
bee6fcfc8d log because i think i deploy bad 2022-01-27 16:05:17 -07:00
bel
fa4dd1a518 unit tests are good 2022-01-27 15:53:12 -07:00
bel
34f16f3259 configurable fastexact radius 2022-01-27 15:49:59 -07:00
bel
adebff5c40 add per broker enable 2022-01-27 15:45:27 -07:00
Bel LaPointe
d7339c855e if parse script to redir, then return no auth 2022-01-27 15:37:58 -07:00
Bel LaPointe
0ab4b795bd return errnoauth on 401-403 for fe 2022-01-27 15:28:00 -07:00
Bel LaPointe
9427e85202 add fastexact to main 2022-01-27 15:25:19 -07:00
Bel LaPointe
5742a215b1 try suppressing logs but we 2022-01-27 15:24:04 -07:00
Bel LaPointe
a742433a56 update config for fastexact, gzip decompress fastexact at parse time 2022-01-27 15:22:04 -07:00
Bel LaPointe
bbe4c4b6ae impl real fe do req 2022-01-27 15:03:07 -07:00
Bel LaPointe
1359be1db4 impl fast exact parse 2022-01-27 15:01:38 -07:00
Bel LaPointe
12fd38ad2c add goquery 2022-01-27 14:19:25 -07:00
Bel LaPointe
37a18acdfe testdata fastexact search response 2022-01-27 14:18:55 -07:00
Bel LaPointe
9ff1302a73 broker hopefully skips "" cookies 2022-01-27 14:08:01 -07:00
Bel LaPointe
99c6c9179a fast exact not done 2022-01-27 14:02:52 -07:00
Bel LaPointe
34fc5fbc5c stub 2022-01-27 14:02:51 -07:00
Bel LaPointe
16bf70a276 ntg vision doesnt export excess 2022-01-27 14:02:37 -07:00
Bel LaPointe
b02f0f28c8 bash scripts show its all cookie phpsessid 2022-01-27 14:02:37 -07:00
17 changed files with 678 additions and 25 deletions

View File

@@ -71,10 +71,16 @@ func setCookies(db storage.DB, host string, resp *http.Response) error {
nameValues := [][2]string{}
old, _ := getCookies(db, host)
for _, value := range old {
if len(value) == 0 {
continue
}
name := strings.Split(value, "=")[0]
nameValues = append(nameValues, [2]string{name, value})
}
for _, value := range resp.Header["Set-Cookie"] {
if len(value) == 0 {
continue
}
name := strings.Split(value, "=")[0]
found := false
for i := range nameValues {
@@ -89,6 +95,9 @@ func setCookies(db storage.DB, host string, resp *http.Response) error {
}
result := []string{}
for i := range nameValues {
if len(nameValues[i][1]) == 0 {
continue
}
result = append(result, nameValues[i][1])
}
logtr.Verbosef("setting cookies for %s: %+v", host, result)

View File

@@ -3,6 +3,7 @@ package broker
import (
"fmt"
"local/storage"
"local/truckstop/logtr"
"net/http"
"net/http/httptest"
"strconv"
@@ -14,12 +15,15 @@ import (
)
func TestBrokerInterface(t *testing.T) {
logtr.SetLevel(logtr.SOS)
var b Broker
b = FastExact{}
b = NTGVision{}
_ = b
}
func TestDoCookies(t *testing.T) {
logtr.SetLevel(logtr.SOS)
limiter = rate.NewLimiter(rate.Limit(20.0), 1)
calls := 0
db := storage.NewMap()

BIN
broker/exec-truckstop Normal file

Binary file not shown.

268
broker/fastexact.go Normal file
View File

@@ -0,0 +1,268 @@
package broker
import (
"bytes"
"compress/gzip"
"errors"
"fmt"
"io"
"io/ioutil"
"local/storage"
"local/truckstop/config"
"local/truckstop/logtr"
"net/http"
"strconv"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
)
type FastExact struct {
doer interface {
doRequest(*http.Request) (*http.Response, error)
}
}
func NewFastExact() FastExact {
fe := FastExact{}
fe.doer = fe
if config.Get().Brokers.FastExact.Mock {
fe = fe.WithMock()
}
return fe
}
func (fe FastExact) WithMock() FastExact {
fe.doer = mockFastExactDoer{}
return fe
}
func (fe FastExact) Search(states []config.State) ([]Job, error) {
jobs, err := fe.search(states)
if err == ErrNoAuth {
if err := fe.login(); err != nil {
return nil, err
}
jobs, err = fe.search(states)
}
return jobs, err
}
func (fe FastExact) login() error {
conf := config.Get()
return fe._login(conf.Brokers.FastExact.Username, conf.Brokers.FastExact.Password, conf.DB())
}
func (fe FastExact) _login(username, password string, db storage.DB) error {
req, err := http.NewRequest(
http.MethodPost,
`https://www.fastexact.com/secure/index.php?page=userLogin`,
strings.NewReader(fmt.Sprintf(
`user_name=%s&user_password=%s&buttonSubmit=Login`,
username,
password,
)),
)
if err != nil {
return err
}
fe.setHeaders(req)
db.Set("cookies_"+req.URL.Host, nil)
resp, err := fe.doer.doRequest(req)
if err != nil {
return err
}
b, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("bad status logging into fast exact: %d: %s", resp.StatusCode, b)
}
return nil
}
func (fe FastExact) setHeaders(req *http.Request) {
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
req.Header.Set("Accept-Encoding", "gzip, deflate, br")
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}
func (fe FastExact) search(states []config.State) ([]Job, error) {
var jobs []Job
for _, state := range states {
subjobs, err := fe.searchOne(state)
if err != nil {
return nil, err
}
jobs = append(jobs, subjobs...)
}
dedupeJobs := map[string]Job{}
for _, job := range jobs {
dedupeJobs[job.UID()] = job
}
result := []Job{}
for _, job := range dedupeJobs {
result = append(result, job)
}
return result, nil
}
func (fe FastExact) searchOne(state config.State) ([]Job, error) {
req, err := fe.newRequest(state)
if err != nil {
return nil, err
}
resp, err := fe.doer.doRequest(req)
logtr.Verbosef("req: %+v => resp: %+v", req, resp)
if err != nil {
return nil, err
}
return fe.parse(resp)
}
func (fe FastExact) newRequest(state config.State) (*http.Request, error) {
zip, ok := config.States[state]
if !ok {
return nil, fmt.Errorf("no configured zip for %s", state)
}
req, err := http.NewRequest(
http.MethodGet,
"https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance="+strconv.Itoa(config.Get().Brokers.FastExact.RadiusMiles)+"&st_loc_zip=8",
nil,
)
if err != nil {
return nil, err
}
fe.setHeaders(req)
return req, nil
}
func (fe FastExact) doRequest(req *http.Request) (*http.Response, error) {
resp, err := do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
b, _ := ioutil.ReadAll(resp.Body)
logtr.Errorf("fastExact bad status: %d: %s", resp.StatusCode, b)
if resp.StatusCode > 400 && resp.StatusCode < 404 {
return nil, ErrNoAuth
}
return nil, fmt.Errorf("bad status from FastExact: %d: %s", resp.StatusCode, b)
}
return resp, nil
}
func (fe FastExact) parse(resp *http.Response) ([]Job, error) {
b, _ := ioutil.ReadAll(resp.Body)
if !bytes.HasPrefix(bytes.TrimSpace(b), []byte("<")) {
gzip, err := gzip.NewReader(bytes.NewReader(b))
if err != nil {
return nil, err
}
b, _ = ioutil.ReadAll(gzip)
}
if bytes.Contains(b, []byte(`<script>window.location.href='index.php'</script>`)) {
return nil, ErrNoAuth
}
logtr.Verbosef("fe.parse %s", b)
resp.Body = io.NopCloser(bytes.NewReader(b))
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return nil, err
}
result := make([]Job, 0)
doc.Find("#list table tr").Each(func(i int, s *goquery.Selection) {
columns := []string{}
s.Find("td").Each(func(i int, s *goquery.Selection) {
if s.Nodes[0].LastChild != nil && len(s.Nodes[0].LastChild.Attr) > 0 {
attrs := s.Nodes[0].LastChild.Attr
columns = append(columns, attrs[len(attrs)-1].Val)
} else {
columns = append(columns, s.Text())
}
})
if len(columns) < 9 {
return
}
job := Job{
ID: columns[0],
URI: columns[8],
}
job.Pickup.Date, _ = time.ParseInLocation("02-Jan-2006 15:04:05", columns[7], time.Local)
job.Pickup.City = strings.Title(strings.ToLower(strings.Split(columns[1], ",")[0]))
if strings.Contains(columns[1], ",") {
job.Pickup.State = strings.Title(strings.Split(strings.Split(columns[1], ",")[1], " ")[1])
}
job.Dropoff.Date = job.Pickup.Date
job.Dropoff.City = strings.Title(strings.ToLower(strings.Split(columns[2], ",")[0]))
if strings.Contains(columns[2], ",") {
job.Dropoff.State = strings.Title(strings.Split(strings.Split(columns[2], ",")[1], " ")[1])
}
job.Miles, _ = strconv.Atoi(columns[3])
if strings.Contains(columns[4], "/") {
weight, _ := strconv.ParseFloat(strings.TrimSpace(strings.Split(columns[4], "/")[1]), 32)
job.Weight = int(weight)
}
job.Meta = fmt.Sprintf(`dimensions:%s`, strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(columns[5], " ", ""), "\n", "")))
logtr.Verbosef("fe.parse %+v => %+v", columns, job)
result = append(result, job)
})
return result, nil
}
type mockFastExactDoer struct{}
func (mock mockFastExactDoer) doRequest(req *http.Request) (*http.Response, error) {
if req.URL.Path != "/secure/index.php" {
return nil, errors.New("bad path")
}
switch req.URL.Query().Get("page") {
case "userLogin":
if b, _ := ioutil.ReadAll(req.Body); !bytes.Equal(b, []byte(`user_name=u&user_password=p&buttonSubmit=Login`)) {
return nil, errors.New("bad req body")
}
return &http.Response{
Status: http.StatusText(http.StatusOK),
StatusCode: http.StatusOK,
Header: http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromLogin; path=/"}},
Body: io.NopCloser(bytes.NewReader([]byte{})),
}, nil
case "ajaxListJobs":
if req.URL.Query().Get("action") != "ajax" {
return nil, errors.New("bad query: action should be ajax")
}
if req.URL.Query().Get("records_per_page") != "50" {
return nil, errors.New("bad query: records_per_page should be 50")
}
if req.URL.Query().Get("distance") != strconv.Itoa(config.Get().Brokers.FastExact.RadiusMiles) {
return nil, errors.New("bad query: distance should be as configured")
}
if req.URL.Query().Get("zipcode") == "" {
return nil, errors.New("bad query: zip code empty")
}
b, err := ioutil.ReadFile("./testdata/fastexact_search.xml")
if err != nil {
b, err = ioutil.ReadFile("./broker/testdata/fastexact_search.xml")
}
if err != nil {
return nil, err
}
return &http.Response{
Status: http.StatusText(http.StatusOK),
StatusCode: http.StatusOK,
Header: http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromSearch; path=/"}},
Body: io.NopCloser(bytes.NewReader(b)),
}, nil
}
return nil, errors.New("bad query")
}

97
broker/fastexact_test.go Normal file
View File

@@ -0,0 +1,97 @@
package broker
import (
"local/storage"
"local/truckstop/config"
"local/truckstop/logtr"
"os"
"path"
"testing"
"golang.org/x/time/rate"
)
func TestFastExactReal(t *testing.T) {
if os.Getenv("INTEGRATION_FAST_EXACT") == "" {
t.Skip("$INTEGRATION_FAST_EXACT not set")
}
os.Setenv("CONFIG", "../config.json")
if err := config.Refresh(nil); err != nil {
t.Fatal(err)
}
conf := config.Get()
conf.Storage = []string{"files", path.Join(t.TempDir(), "storage")}
os.Setenv("CONFIG", path.Join(t.TempDir(), "config.json"))
config.Set(*conf)
if err := config.Refresh(nil); err != nil {
t.Fatal(err)
}
states := []config.State{
config.State("NC"),
}
fe := NewFastExact()
if err := fe.login(); err != nil {
t.Fatal(err)
}
jobs, err := fe.search(states)
if err != nil {
t.Fatal(err)
}
t.Logf("%d", len(jobs))
for i := range jobs {
t.Logf("[%d] %+v", i, jobs[i])
}
}
func TestFastExactLogin(t *testing.T) {
logtr.SetLevel(logtr.SOS)
limiter = rate.NewLimiter(rate.Limit(60.0), 1)
fe := NewFastExact().WithMock()
_ = fe
db := storage.NewMap()
if err := fe._login("u", "p", db); err != nil {
t.Fatal(err)
}
}
func TestFastExactSearch(t *testing.T) {
logtr.SetLevel(logtr.SOS)
limiter = rate.NewLimiter(rate.Limit(60.0), 1)
fe := NewFastExact().WithMock()
_ = fe
db := storage.NewMap()
_ = db
if jobs, err := fe.search([]config.State{config.State("NC"), config.State("SC")}); err != nil {
t.Fatal(err)
} else if len(jobs) != 10 {
t.Fatal(len(jobs))
} else {
for _, job := range jobs {
if job.ID == "" {
t.Error(job)
} else if job.Miles == 0 {
t.Error(job)
} else if job.URI == "" {
t.Error(job)
} else if job.Meta == "" {
t.Error(job)
} else if job.Weight == 0 {
t.Error(job)
} else if job.Pickup.Date.IsZero() {
t.Error(job)
} else if job.Pickup.Date != job.Dropoff.Date {
t.Error(job)
} else if job.Dropoff.State == "" {
t.Error(job)
} else if job.Dropoff.City == "" {
t.Error(job)
} else if job.Pickup.State == "" {
t.Error(job)
} else if job.Pickup.City == "" {
t.Error(job)
}
}
}
}

View File

@@ -16,7 +16,7 @@ import (
type NTGVision struct {
searcher interface {
search(states []config.State) (io.ReadCloser, error)
searchJob(id int64) (io.ReadCloser, error)
searchJobReadCloser(id int64) (io.ReadCloser, error)
}
}
@@ -96,7 +96,7 @@ func (ntgJob *ntgVisionJob) JobInfo() (ntgVisionJobInfo, error) {
return ntgJob.jobinfo, nil
}
ntg := NewNTGVision()
ji, err := ntg.SearchJob(ntgJob.ID)
ji, err := ntg.searchJob(ntgJob.ID)
if err == nil {
ntgJob.jobinfo = ji
b, err := json.Marshal(ntgJob.jobinfo)
@@ -107,7 +107,7 @@ func (ntgJob *ntgVisionJob) JobInfo() (ntgVisionJobInfo, error) {
return ji, err
}
func (ntg NTGVision) searchJob(id int64) (io.ReadCloser, error) {
func (ntg NTGVision) searchJobReadCloser(id int64) (io.ReadCloser, error) {
time.Sleep(config.Get().Interval.JobInfo.Get())
request, err := http.NewRequest(http.MethodGet, fmt.Sprintf(config.Get().Brokers.NTG.LoadPageAPIURIFormat, id), nil)
if err != nil {
@@ -169,8 +169,8 @@ func (ntg NTGVision) WithMock() NTGVision {
return ntg
}
func (ntg NTGVision) SearchJob(id int64) (ntgVisionJobInfo, error) {
rc, err := ntg.searcher.searchJob(id)
func (ntg NTGVision) searchJob(id int64) (ntgVisionJobInfo, error) {
rc, err := ntg.searcher.searchJobReadCloser(id)
if err != nil {
return ntgVisionJobInfo{}, err
}

View File

@@ -21,7 +21,7 @@ func (ntgm NTGVisionMock) search(states []config.State) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(b)), err
}
func (ntgm NTGVisionMock) searchJob(id int64) (io.ReadCloser, error) {
func (ntgm NTGVisionMock) searchJobReadCloser(id int64) (io.ReadCloser, error) {
path := path.Join(os.Getenv("GOPATH"), "src", "local", "truckstop", "broker", "testdata", "ntgvision_jobinfo_response.json")
b, err := ioutil.ReadFile(path)
return io.NopCloser(bytes.NewReader(b)), err

12
broker/testdata/fastexact_login.sh vendored Normal file
View File

@@ -0,0 +1,12 @@
#! /bin/bash
curl -i -Ss \
'https://www.fastexact.com/secure/index.php?page=userLogin' \
-X POST \
-H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' \
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \
-H 'Accept-Language: en-US,en;q=0.5' \
-H 'Accept-Encoding: gzip, deflate, br' \
-H 'Content-Type: application/x-www-form-urlencoded' \
--data-raw 'user_name=birdman&user_password=166647&buttonSubmit=Login' \

10
broker/testdata/fastexact_search.sh vendored Normal file
View File

@@ -0,0 +1,10 @@
#! /bin/bash
curl -sS \
'https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode=27006&records_per_page=10&distance=300&st_loc_zip=8' \
-H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' \
-H 'Accept: */*' \
-H 'Accept-Language: en-US,en;q=0.5' \
-H 'Accept-Encoding: gzip, deflate, br' \
-H 'Content-Type: application/x-www-form-urlencoded' \
-H 'Cookie: PHPSESSID=vq4ss3lbfg0viluaau7e0d32q6' \
| gzip -d

218
broker/testdata/fastexact_search.xml vendored Normal file
View File

@@ -0,0 +1,218 @@
<form name="frm_search" action="index.php?page=manageJobs" method="post">
<a href="https://play.google.com/store/apps/details?id=app.FastExact" target="_blank" title="Android app, Available in the App Store, Download and get all more..." class="android_app"></a>
<a href="http://itunes.apple.com/us/app/fastexact/id497933887?mt=8" target="_blank" title="iPhone app, Available in the App Store, Download and get all more..." class="iphone_app"></a>
<div class="search_box w67p">
<div class="header"><strong>Search</strong> <span class="usZipLink"><a href="http://zip4.usps.com/zip4/citytown.jsp" title="US Zip Code Finder" target="_blank">US Zip Code Finder &raquo;</a></span> </div>
<div class="content">
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td valign="top" width="342"><strong>Enter your Zip Code to show radius
<!--of 100 miles-->
from your location</strong></td>
<td valign="top" width="68"><input type="text" name="zipcode" id="zipcode" value="27006" onKeyPress="return disableEnterKey(event)" onkeydown="if(event.keyCode == 13){radiusSearch('10')}" /></td>
<td valign="top" width="59"><select name="distance" id="distance" class="select" onkeydown="if(event.keyCode == 13){radiusSearch('10')}">
<option value="100" >100</option>
<option value="200" >200</option>
<option value="300" selected=selected>300</option>
</select></td>
<td valign="top" width="128"><!--<input type="checkbox" name="st_loc_zip" id="st_loc_zip" value="1" class="ownBox" checked=checked onKeyPress="return disableEnterKey(event)" onkeydown="if(event.keyCode == 13){radiusSearch('10')}"/>--></td>
<td valign="top" class="padr12" width="337">Make this your current zip code<br />
for the next
<select name="st_loc_zip" id="st_loc_zip" class="dropdown">
<option value="1">1</option>
<option value="2">2</option>
<option value="3">3</option>
<option value="4">4</option>
<option value="5">5</option>
<option value="6">6</option>
<option value="7">7</option>
<option value="8" selected="selected">8</option>
<option value="9">9</option>
<option value="10">10</option>
<option value="11">11</option>
<option value="12">12</option>
<option value="13">13</option>
<option value="14">14</option>
<option value="15">15</option>
<option value="16">16</option>
<option value="17">17</option>
<option value="18">18</option>
<option value="19">19</option>
<option value="20">20</option>
<option value="21">21</option>
<option value="22">22</option>
<option value="23">23</option>
<option value="24">24</option>
</select>
hours.<a href="javascript:radiusSearch('10','save');" ><img src="images/buttons/save.png" /></a></td>
<td valign="top"><a href="javascript:radiusSearch('10','search');"><img src="images/buttons/search.gif" alt="Search" align="middle" title="Search" /></a></td>
</tr>
</table>
</div>
</div>
<div class="br"></div>
</form>
<br clear="all" />
<div id="list" align="center">
<table width="100%" border="0" cellspacing="0" cellpadding="0" summary="Order List">
<tr>
<th width="14%" height="38" align="center">ID&nbsp;&nbsp;<span id="sort8"><a href="javascript:contentSort('ordernum=8&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',8)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
<th width="14%" height="38" align="center">Origin&nbsp;&nbsp;<span id="sort1"><a href="javascript:contentSort('ordernum=1&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',1)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
<th width="14%" height="38" align="center">Destination&nbsp;&nbsp;<span id="sort2"><a href="javascript:contentSort('ordernum=2&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',2)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
<th width="14%" height="38" align="center">Mile&nbsp;&nbsp;<span id="sort3"><a href="javascript:contentSort('ordernum=3&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',3)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
<th height="38" align="center">Pieces / Weight&nbsp;&nbsp;<span id="sort4"><a href="javascript:contentSort('ordernum=4&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',4)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0"/></a></span></th>
<th width="14%" height="38" align="center">Dimension <span id="sort5"><a href="javascript:contentSort('ordernum=5&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',5)"><img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0"/></a></span></th>
<th width="14%" height="38" align="center">Vehicle Size <span id="sort5"><a href="javascript:contentSort('ordernum=5&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',5)"><img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0"/></a></span></th>
<th width="20%" height="38" align="center">Pickup (EST)&nbsp;&nbsp;<span id="sort6"><a href="javascript:contentSort('ordernum=6&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',6)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
<th width="6%" height="38" align="center">View</th>
</tr>
<tr >
<td align="center" height="38">Z-5344222</td>
<td align="center" height="38">Circleville, OH 43113</td>
<td align="center">Willard, OH 44890</td>
<td align="center">128</td>
<td align="center">4 / 4800</td>
<td align="center">96x48x88in.</td>
<td align="center">SMALL STRAIGHT</td>
<td align="center">31-Jan-2022 17:00:00</td>
<td align="center"><!--thickbox-->
<a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344222"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
</tr>
<tr class="alternate">
<td align="center" height="38">Z-5344003</td>
<td align="center" height="38">WASHINGTON, DC</td>
<td align="center">WINSTON SALEM, NC</td>
<td align="center">332</td>
<td align="center">2 / 3000</td>
<td align="center">78x35x99 in.</td>
<td align="center">LARGE STRAIGHT</td>
<td align="center">31-Jan-2022 08:00:00</td>
<td align="center"><!--thickbox-->
<a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344003"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
</tr>
<tr >
<td align="center" height="38">Z-5344092</td>
<td align="center" height="38">Winchester, VA 22603</td>
<td align="center">Milwaukee, WI 53221</td>
<td align="center">728</td>
<td align="center">4 / 2195</td>
<td align="center">NO DIMENSIONS SPECIFIED</td>
<td align="center">VAN</td>
<td align="center">30-Jan-2022 15:00:00</td>
<td align="center"><!--thickbox-->
<a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344092"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
</tr>
<tr class="alternate">
<td align="center" height="38">Z-5344035</td>
<td align="center" height="38">ATLANTA, GA 30301</td>
<td align="center">BURNSVILLE, MN 55337</td>
<td align="center">1098</td>
<td align="center">10 / 6000</td>
<td align="center">48x40x40 in.</td>
<td align="center">LARGE STRAIGHT</td>
<td align="center">29-Jan-2022 08:00:00</td>
<td align="center"><!--thickbox-->
<a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344035"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
</tr>
<tr >
<td align="center" height="38">Z-5344518</td>
<td align="center" height="38">Hamlet, NC 28345</td>
<td align="center">East Windsor, CT 06088</td>
<td align="center">720</td>
<td align="center">1 / 500</td>
<td align="center">54x36x24in.</td>
<td align="center">CARGO VAN</td>
<td align="center">28-Jan-2022 13:00:00</td>
<td align="center"><!--thickbox-->
<a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344518"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
</tr>
<tr class="alternate">
<td align="center" height="38">X-5344304</td>
<td align="center" height="38">MOUNT PLEASANT, PA 15666</td>
<td align="center">MIAMI, FL 33178-1056</td>
<td align="center">1156</td>
<td align="center">0 / 5000.00</td>
<td align="center">48Inches x 48Inches x </td>
<td align="center">Straight Truck 20-24 ft</td>
<td align="center">28-Jan-2022 13:00:00</td>
<td align="center"><!--thickbox-->
<a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344304"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
</tr>
<tr >
<td align="center" height="38">Z-5344061</td>
<td align="center" height="38">Chesapeake, VA 23321</td>
<td align="center">Fort Wayne, IN 46818</td>
<td align="center">727</td>
<td align="center">0 / 5000</td>
<td align="center">NO DIMENSIONS SPECIFIED</td>
<td align="center">LARGE STRAIGHT</td>
<td align="center">28-Jan-2022 13:00:00</td>
<td align="center"><!--thickbox-->
<a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344061"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
</tr>
<tr class="alternate">
<td align="center" height="38">B-5344514</td>
<td align="center" height="38">MAXTON, NC 28364</td>
<td align="center">PEMBINA, ND 58271</td>
<td align="center">1637</td>
<td align="center">7 / 28000</td>
<td align="center">7@54x96x40</td>
<td align="center"></td>
<td align="center">28-Jan-2022 12:00:00</td>
<td align="center"><!--thickbox-->
<a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344514"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
</tr>
<tr >
<td align="center" height="38">Z-5344114</td>
<td align="center" height="38">Durham, NC 27703</td>
<td align="center">Cumming, GA 30040</td>
<td align="center">368</td>
<td align="center">6 / 3200</td>
<td align="center">48x40x48in.</td>
<td align="center">LARGE STRAIGHT</td>
<td align="center">28-Jan-2022 12:00:00</td>
<td align="center"><!--thickbox-->
<a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344114"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
</tr>
<tr class="alternate">
<td align="center" height="38">Z-5344231</td>
<td align="center" height="38">Durham, NC 27702</td>
<td align="center">Cumming, GA 30040</td>
<td align="center">366</td>
<td align="center">6 / 3200</td>
<td align="center">48x40x48in.</td>
<td align="center">LARGE STRAIGHT</td>
<td align="center">28-Jan-2022 12:00:00</td>
<td align="center"><!--thickbox-->
<a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344231"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
</tr>
</table>
</div>
<br clear="all" />
<div class="paging">
<table cellspacing="0" cellpadding="0" border="0" width="100%">
<tbody>
<tr>
<td>Details Found: 81<img height="1" width="14" alt="" src="images/tspacer.gif">Page 1 of 9</td>
<td align="center"><div id="Page">
<img height="1" width="8" alt="" src="images/tspacer.gif">
<a href="javascript:fnPaging('pageNum=2&records_per_page=10&zipcode=27006&distance=300&st_loc_zip=8')">Next</a> <strong>&raquo;</strong>
</div></td>
<td align="right"><table cellspacing="0" cellpadding="0" border="0">
<tbody>
<tr>
<td valign="top" class="record">Records Per Page:&nbsp;</td>
<td valign="top"><div id="recNum">
<select style="font-size: 11px;" class="dropdown" id="pagei" name="pagei" onchange="changeRecordCount('27006','300','8')">
<option value="10" selected="selected" >10</option>
<option value="20" >20</option>
<option value="50" >50</option>
<option value="100" >100</option>
</select>
</div></td>
</tr>
</tbody>
</table></td>
</tr>
</tbody>
</table>

View File

@@ -34,7 +34,7 @@
"Pickup": false,
"Dropoff": false,
"Pathed": {
"Enabled": true,
"Enabled": false,
"DirectionsURIFormat": "https://maps.googleapis.com/maps/api/directions/json?origin=%s\u0026destination=%s\u0026mode=driving\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
"PathedURIFormat": "https://maps.googleapis.com/maps/api/staticmap?size=250x250\u0026path=%s\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg\u0026size=250x250\u0026markers=%s|%s",
"Zoom": {
@@ -69,9 +69,17 @@
"Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
}
},
"Once": true,
"Once": false,
"Brokers": {
"FastExact": {
"RadiusMiles": 100,
"Enabled": true,
"Mock": true,
"Username": "birdman",
"Password": "166647"
},
"NTG": {
"Enabled": false,
"JobInfo": true,
"Mock": true,
"LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",

View File

@@ -71,7 +71,15 @@
},
"Once": true,
"Brokers": {
"FastExact": {
"RadiusMiles": 100,
"Enabled": true,
"Mock": true,
"Username": "u",
"Password": "p"
},
"NTG": {
"Enabled": true,
"JobInfo": true,
"Mock": true,
"LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",

View File

@@ -65,6 +65,7 @@ type Config struct {
Once bool
Brokers struct {
NTG struct {
Enabled bool
JobInfo bool
Mock bool
LoadPageURIFormat string
@@ -73,9 +74,11 @@ type Config struct {
Password string
}
FastExact struct {
Mock bool
Username string
Password string
Enabled bool
RadiusMiles int
Mock bool
Username string
Password string
}
}

2
go.mod
View File

@@ -16,8 +16,10 @@ require (
require (
cloud.google.com/go v0.33.1 // indirect
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 // indirect
github.com/abbot/go-http-auth v0.4.0 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/aws/aws-sdk-go v1.15.81 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect

5
go.sum
View File

@@ -7,12 +7,16 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 h1:6X8iB881g299aNEv6KXrcjL31iLOH7yA6NXoQX+MbDg=
github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
github.com/a8m/tree v0.0.0-20180321023834-3cf936ce15d6/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=
github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
github.com/anacrolix/dms v0.0.0-20180117034613-8af4925bffb5/go.mod h1:DGqLjaZ3ziKKNRt+U5Q9PLWJ52Q/4rxfaaH/b3QYKaE=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/aws/aws-sdk-go v1.15.81 h1:va7uoFaV9uKAtZ6BTmp1u7paoMsizYRRLvRuoC07nQ8=
github.com/aws/aws-sdk-go v1.15.81/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
github.com/billziss-gh/cgofuse v1.1.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
@@ -219,6 +223,7 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=

26
main.go
View File

@@ -303,7 +303,7 @@ func once() error {
if ok, err := sendJob(jobs[i]); err != nil {
return err
} else if ok {
logtr.Debugf("sent job", jobs[i])
logtr.Debugf("sent job %+v", jobs[i])
if err := db.Set(jobs[i].UID(), []byte(`sent`)); err != nil {
return err
}
@@ -314,8 +314,16 @@ func once() error {
func getJobs() ([]broker.Job, error) {
states := config.AllStates()
ntg := broker.NewNTGVision()
brokers := []broker.Broker{ntg}
brokers := []broker.Broker{}
if config.Get().Brokers.NTG.Enabled {
logtr.Debugf("NTG enabled")
brokers = append(brokers, broker.NewNTGVision())
}
if config.Get().Brokers.FastExact.Enabled {
logtr.Debugf("FastExact enabled")
brokers = append(brokers, broker.NewFastExact())
}
logtr.Debugf("brokers=%+v", brokers)
jobs := []broker.Job{}
for _, broker := range brokers {
somejobs, err := broker.Search(states)
@@ -344,7 +352,7 @@ func updateDeadJobs(jobs []broker.Job) error {
wouldBe := strings.TrimPrefix(listEntry, "sent_job_")
found := false
for i := range jobs {
found = found || jobs[i].ID == wouldBe
found = found || jobs[i].UID() == wouldBe || jobs[i].ID == wouldBe
}
logtr.Debugf("found job %s to be still alive==%v", wouldBe, found)
if !found {
@@ -363,14 +371,14 @@ func updateDeadJobs(jobs []broker.Job) error {
}
*/
if err := message.NewMatrix().Remove(recorded.MatrixID); err != nil {
return err
logtr.Debugf("failed to remove matrix: %v: %v", recorded.MatrixID, err)
}
if err := db.Set(listEntry, nil); err != nil {
return err
logtr.Debugf("failed to remove db: %v: %v", listEntry, err)
}
for _, imageid := range recorded.MatrixImageIDs {
if err := message.NewMatrix().Remove(imageid); err != nil {
return err
logtr.Debugf("failed to remove matrix image: %v: %v", imageid, err)
}
}
}
@@ -399,7 +407,7 @@ func dropBanlistJobs(jobs []broker.Job) ([]broker.Job, error) {
func sendJob(job broker.Job) (bool, error) {
sender := message.NewMatrix()
payload := job.FormatMultilineText()
logtr.Debugf("once: send job %s if nonzero: %s", job.String(), payload)
logtr.Debugf("once: send job %s if nonzero: %q", job.String(), payload)
if len(payload) == 0 {
return false, nil
}
@@ -419,7 +427,7 @@ func sendJob(job broker.Job) (bool, error) {
logtr.Errorf("failed to marshal recorded job: %v", err)
return
}
if err := db.Set("sent_job_"+job.ID, b); err != nil {
if err := db.Set("sent_job_"+job.UID(), b); err != nil {
logtr.Errorf("failed to set recorded job: %v", err)
return
}

View File

@@ -1,20 +1,21 @@
todo:
- !states emits current state
- TEST. Just like, refactor and test to shit.
- try search ntg by autoinc?
- fast exact is dumb or...?
- modify old items once no longer available; drop stale jobs good candidate but requires new matrix interaction
- more than NTG; blue one
- !states emits current state
- test each !command callbacks to matrix
- recv-as for clients so pa receives mas commands as writes
- continuation is garbo, but I can still do better client side to avoid get-set high level
- no hard code jpeg or have it in multiple places
- change matrix so I test my custom logic even if I dont fetch remote
- warn/err/etc. on clobbering ids.matrix since clients can mess with one another
- modify old items once no longer available; drop stale jobs good candidate but requires new matrix interaction
- more than NTG
- todo: filter out jobs like CA
subtasks:
- banlist criteria like vendors, brokers, metadata
- set up copy for caleb, broc
done:
- try search ntg by autoinc?
- TEST. Just like, refactor and test to shit.
- mark jobs no longer avail by modifying in matrix;; save matrix ID over dummy payload
- TEST its falling apart
- help() log on truckstop for stuff like perma 403