From faf253982516678b3a685042e127dcd52f59fd84 Mon Sep 17 00:00:00 2001 From: Bel LaPointe Date: Mon, 10 Jan 2022 20:44:07 -0500 Subject: [PATCH] impl ntg auth refresh-cache --- broker/ntgvision.go | 70 ++++++++++++++++++++++++++--- broker/testdata/ntg.sh | 23 ++++++++++ broker/testdata/ntgvision_search.sh | 5 ++- config.json | 10 +++-- config/config.go | 6 ++- testdata/scrape_cookies.sh | 3 +- 6 files changed, 103 insertions(+), 14 deletions(-) create mode 100644 broker/testdata/ntg.sh diff --git a/broker/ntgvision.go b/broker/ntgvision.go index 720f1c0..c793656 100644 --- a/broker/ntgvision.go +++ b/broker/ntgvision.go @@ -3,12 +3,12 @@ package broker import ( "bytes" "encoding/json" + "errors" "fmt" "io" "io/ioutil" "local/truckstop/config" "net/http" - "strings" "time" ) @@ -18,6 +18,8 @@ type NTGVision struct { } } +var ErrNoAuth = errors.New("not authorized") + type ntgVisionJob struct { ID int64 `json:"id"` PickupDate string `json:"sDate"` @@ -82,6 +84,56 @@ func (ntg NTGVision) Search(states []config.State) ([]Job, error) { } func (ntg NTGVision) search(states []config.State) (io.ReadCloser, error) { + if config.Get().Brokers.NTG.Token == "" { + if err := ntg.refreshAuth(); err != nil { + return nil, err + } + } + rc, err := ntg._search(states) + if err == ErrNoAuth { + if err := ntg.refreshAuth(); err != nil { + return nil, err + } + rc, err = ntg._search(states) + } + return rc, err +} + +func (ntg NTGVision) refreshAuth() error { + b, _ := json.Marshal(map[string]string{ + "username": config.Get().Brokers.NTG.Username, + "password": config.Get().Brokers.NTG.Password, + }) + request, err := http.NewRequest(http.MethodPost, "https://ntgvision.com/api/v1/sts/Login", bytes.NewReader(b)) + if err != nil { + return err + } + setNTGHeaders(request) + resp, err := http.DefaultClient.Do(request) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("failed refreshing token: (%d): %s", resp.StatusCode, b) + } + var v struct { + Token string `json:"token"` + } + if err := json.NewDecoder(resp.Body).Decode(&v); err != nil { + return err + } + if len(v.Token) == 0 { + return errors.New("failed to get token from login call") + } + conf := config.Get() + conf.Brokers.NTG.Token = v.Token + config.Set(*conf) + return nil +} + +func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) { request, err := ntg.newRequest(states) if err != nil { return nil, err @@ -94,6 +146,9 @@ func (ntg NTGVision) search(states []config.State) (io.ReadCloser, error) { if resp.StatusCode != http.StatusOK { b, _ := ioutil.ReadAll(resp.Body) resp.Body.Close() + if resp.StatusCode > 400 && resp.StatusCode < 404 { + return nil, ErrNoAuth + } return nil, fmt.Errorf("bad status searching ntg: %d: %s", resp.StatusCode, b) } return resp.Body, nil @@ -123,21 +178,26 @@ func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) { if err != nil { return nil, err } + setNTGHeaders(request) + request.Header.Set("Authorization", "Bearer "+config.Get().Brokers.NTG.Token) + + return request, nil +} + +func setNTGHeaders(request *http.Request) { request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0") request.Header.Set("Accept", "application/json, text/plain, */*") request.Header.Set("Accept-Language", "en-US,en;q=0.5") request.Header.Set("Accept-Encoding", "gzip, deflate, br") request.Header.Set("Content-Type", "application/json;charset=utf-8") - request.Header.Set("Authorization", "Bearer "+strings.Split(strings.Split(config.Get().Brokers.NTG.Cookie, "; NTGAuthToken=")[1], "; ")[0]) + //request.Header.Set("Authorization", "Bearer "+config.Get().Brokers.NTG.Token) request.Header.Set("Origin", "https://ntgvision.com") request.Header.Set("DNT", "1") request.Header.Set("Connection", "keep-alive") - request.Header.Set("Cookie", config.Get().Brokers.NTG.Cookie) + //request.Header.Set("Cookie", config.Get().Brokers.NTG.Cookie) request.Header.Set("Sec-Fetch-Dest", "empty") request.Header.Set("Sec-Fetch-Mode", "cors") request.Header.Set("Sec-Fetch-Site", "same-origin") request.Header.Set("Pragma", "no-cache") request.Header.Set("Cache-Control", "no-cache") - - return request, nil } diff --git a/broker/testdata/ntg.sh b/broker/testdata/ntg.sh new file mode 100644 index 0000000..079af26 --- /dev/null +++ b/broker/testdata/ntg.sh @@ -0,0 +1,23 @@ +#! /bin/bash + +login() { +curl 'https://ntgvision.com/api/v1/sts/Login' -X POST -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:95.0) Gecko/20100101 Firefox/95.0' -H 'Accept: application/json, text/plain, */*' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'Content-Type: application/json;charset=utf-8' -H 'Origin: https://ntgvision.com' -H 'Connection: keep-alive' -H 'Cookie: _ga=GA1.2.638105752.1641827398; _gid=GA1.2.730578667.1641827398; cookiesession1=678A3E13IJKLMNOPQRSTUVWXYZAC7685' -H 'Sec-Fetch-Dest: empty' -H 'Sec-Fetch-Mode: cors' -H 'Sec-Fetch-Site: same-origin' --data-raw '{"username":"noeasyrunstrucking@gmail.com","password":"thumper123"}' +} + +login_2() { +curl 'https://ntgvision.com/api/v1/sts/Login' \ + -X POST \ + -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:95.0) Gecko/20100101 Firefox/95.0' \ + -H 'Accept: application/json, text/plain, */*' \ + -H 'Accept-Language: en-US,en;q=0.5' \ + -H 'Accept-Encoding: gzip, deflate, br' \ + -H 'Content-Type: application/json;charset=utf-8' \ + -H 'Origin: https://ntgvision.com' \ + -H 'Connection: keep-alive' \ + -H 'Sec-Fetch-Dest: empty' \ + -H 'Sec-Fetch-Mode: cors' \ + -H 'Sec-Fetch-Site: same-origin' \ + -d '{"username":"noeasyrunstrucking@gmail.com","password":"thumper123"}' +} + +login_2 diff --git a/broker/testdata/ntgvision_search.sh b/broker/testdata/ntgvision_search.sh index f329c71..b987d24 100644 --- a/broker/testdata/ntgvision_search.sh +++ b/broker/testdata/ntgvision_search.sh @@ -1,5 +1,7 @@ #! /bin/bash + #-H 'Cookie: cookiesession1=678A3E1398901234BCDEFGHIJKLMA492; NTGAuthToken=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMzgyMSIsInVuaXF1ZV9uYW1lIjoiYmlyZGNvbXBhbnlsb2dpc3RpY3NAZ21haWwuY29tIiwianRpIjoiODRiNWM5ODItMjc0ZS00YmQwLTg1NjktYmFjYWQ1OWVmOTRhIiwiaWF0IjoiMS8xMC8yMDIyIDI6MjU6MjcgQU0iLCJudGd2Um9sZSI6IkNhcnJpZXJBcHByb3ZlZCIsImxvY2tVc2VyIjoiRmFsc2UiLCJvdmVycmlkZUJsYWNrbGlzdCI6IkZhbHNlIiwic2hvd1JhdGVzIjoiRmFsc2UiLCJ1c2VyQ2FycmllcnMiOiIxMTQxOTMiLCJvdHJVc2VyIjoiRmFsc2UiLCJuYmYiOjE2NDE3ODE1MjcsImV4cCI6MTY0MTg2NDMyNywiaXNzIjoiTlRHIFNlY3VyaXR5IFRva2VuIFNlcnZpY2UiLCJhdWQiOiJOVEcifQ.BfkdMsDlzjJOZJLju00mkmY8efZewV-CuYuq0el6dsiDH6UnlnuPL286QcpzKmi5642WCvneWfaFiDgj92HmiDSFTtaMYPTVI8cvX6TxQ2zs88RyNZ6FzA1ga0xLu0x6VjDrkSS1VAk7mi4xmttnsaUSnUsQ85dbEyXRG_k1M9EHRq17rcCEz8t-s3u4S9JzPRRHmHCEVe8m6R13jAJSf70MVNUiRG_7zIzfIBJJpv9cLkrp-WmtPwjuabUE9lBtfvewBmevT6ZGlfljjpBa-W6I6dO8qU-6TKV3UaBXiYZoGC_YGzT_9VHZ_TC9GnH3oB69EeIm8exwQLthOJcP3w; _uiq_id.711119701.3d83=d0781b15ed3c1fb3.1641781528.0.1641781528..' \ + curl \ 'https://ntgvision.com/api/v1/load/LoadBoardSearchResults' \ -X POST \ @@ -8,11 +10,10 @@ curl \ -H 'Accept-Language: en-US,en;q=0.5' \ -H 'Accept-Encoding: gzip, deflate, br' \ -H 'Content-Type: application/json;charset=utf-8' \ - -H 'Authorization: Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMzgyMSIsInVuaXF1ZV9uYW1lIjoiYmlyZGNvbXBhbnlsb2dpc3RpY3NAZ21haWwuY29tIiwianRpIjoiODRiNWM5ODItMjc0ZS00YmQwLTg1NjktYmFjYWQ1OWVmOTRhIiwiaWF0IjoiMS8xMC8yMDIyIDI6MjU6MjcgQU0iLCJudGd2Um9sZSI6IkNhcnJpZXJBcHByb3ZlZCIsImxvY2tVc2VyIjoiRmFsc2UiLCJvdmVycmlkZUJsYWNrbGlzdCI6IkZhbHNlIiwic2hvd1JhdGVzIjoiRmFsc2UiLCJ1c2VyQ2FycmllcnMiOiIxMTQxOTMiLCJvdHJVc2VyIjoiRmFsc2UiLCJuYmYiOjE2NDE3ODE1MjcsImV4cCI6MTY0MTg2NDMyNywiaXNzIjoiTlRHIFNlY3VyaXR5IFRva2VuIFNlcnZpY2UiLCJhdWQiOiJOVEcifQ.BfkdMsDlzjJOZJLju00mkmY8efZewV-CuYuq0el6dsiDH6UnlnuPL286QcpzKmi5642WCvneWfaFiDgj92HmiDSFTtaMYPTVI8cvX6TxQ2zs88RyNZ6FzA1ga0xLu0x6VjDrkSS1VAk7mi4xmttnsaUSnUsQ85dbEyXRG_k1M9EHRq17rcCEz8t-s3u4S9JzPRRHmHCEVe8m6R13jAJSf70MVNUiRG_7zIzfIBJJpv9cLkrp-WmtPwjuabUE9lBtfvewBmevT6ZGlfljjpBa-W6I6dO8qU-6TKV3UaBXiYZoGC_YGzT_9VHZ_TC9GnH3oB69EeIm8exwQLthOJcP3w' \ + -H "Authorization: Bearer ${TOKEN:-"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMzgyMSIsInVuaXF1ZV9uYW1lIjoiYmlyZGNvbXBhbnlsb2dpc3RpY3NAZ21haWwuY29tIiwianRpIjoiODRiNWM5ODItMjc0ZS00YmQwLTg1NjktYmFjYWQ1OWVmOTRhIiwiaWF0IjoiMS8xMC8yMDIyIDI6MjU6MjcgQU0iLCJudGd2Um9sZSI6IkNhcnJpZXJBcHByb3ZlZCIsImxvY2tVc2VyIjoiRmFsc2UiLCJvdmVycmlkZUJsYWNrbGlzdCI6IkZhbHNlIiwic2hvd1JhdGVzIjoiRmFsc2UiLCJ1c2VyQ2FycmllcnMiOiIxMTQxOTMiLCJvdHJVc2VyIjoiRmFsc2UiLCJuYmYiOjE2NDE3ODE1MjcsImV4cCI6MTY0MTg2NDMyNywiaXNzIjoiTlRHIFNlY3VyaXR5IFRva2VuIFNlcnZpY2UiLCJhdWQiOiJOVEcifQ.BfkdMsDlzjJOZJLju00mkmY8efZewV-CuYuq0el6dsiDH6UnlnuPL286QcpzKmi5642WCvneWfaFiDgj92HmiDSFTtaMYPTVI8cvX6TxQ2zs88RyNZ6FzA1ga0xLu0x6VjDrkSS1VAk7mi4xmttnsaUSnUsQ85dbEyXRG_k1M9EHRq17rcCEz8t-s3u4S9JzPRRHmHCEVe8m6R13jAJSf70MVNUiRG_7zIzfIBJJpv9cLkrp-WmtPwjuabUE9lBtfvewBmevT6ZGlfljjpBa-W6I6dO8qU-6TKV3UaBXiYZoGC_YGzT_9VHZ_TC9GnH3oB69EeIm8exwQLthOJcP3w"}" \ -H 'Origin: https://ntgvision.com' \ -H 'DNT: 1' \ -H 'Connection: keep-alive' \ - -H 'Cookie: cookiesession1=678A3E1398901234BCDEFGHIJKLMA492; NTGAuthToken=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMzgyMSIsInVuaXF1ZV9uYW1lIjoiYmlyZGNvbXBhbnlsb2dpc3RpY3NAZ21haWwuY29tIiwianRpIjoiODRiNWM5ODItMjc0ZS00YmQwLTg1NjktYmFjYWQ1OWVmOTRhIiwiaWF0IjoiMS8xMC8yMDIyIDI6MjU6MjcgQU0iLCJudGd2Um9sZSI6IkNhcnJpZXJBcHByb3ZlZCIsImxvY2tVc2VyIjoiRmFsc2UiLCJvdmVycmlkZUJsYWNrbGlzdCI6IkZhbHNlIiwic2hvd1JhdGVzIjoiRmFsc2UiLCJ1c2VyQ2FycmllcnMiOiIxMTQxOTMiLCJvdHJVc2VyIjoiRmFsc2UiLCJuYmYiOjE2NDE3ODE1MjcsImV4cCI6MTY0MTg2NDMyNywiaXNzIjoiTlRHIFNlY3VyaXR5IFRva2VuIFNlcnZpY2UiLCJhdWQiOiJOVEcifQ.BfkdMsDlzjJOZJLju00mkmY8efZewV-CuYuq0el6dsiDH6UnlnuPL286QcpzKmi5642WCvneWfaFiDgj92HmiDSFTtaMYPTVI8cvX6TxQ2zs88RyNZ6FzA1ga0xLu0x6VjDrkSS1VAk7mi4xmttnsaUSnUsQ85dbEyXRG_k1M9EHRq17rcCEz8t-s3u4S9JzPRRHmHCEVe8m6R13jAJSf70MVNUiRG_7zIzfIBJJpv9cLkrp-WmtPwjuabUE9lBtfvewBmevT6ZGlfljjpBa-W6I6dO8qU-6TKV3UaBXiYZoGC_YGzT_9VHZ_TC9GnH3oB69EeIm8exwQLthOJcP3w; _uiq_id.711119701.3d83=d0781b15ed3c1fb3.1641781528.0.1641781528..' \ -H 'Sec-Fetch-Dest: empty' \ -H 'Sec-Fetch-Mode: cors' \ -H 'Sec-Fetch-Site: same-origin' \ diff --git a/config.json b/config.json index 7f4ae76..39e47d5 100644 --- a/config.json +++ b/config.json @@ -1,5 +1,5 @@ { - "Interval": "1m", + "Interval": 60000000000, "States": [ "GA" ], @@ -11,10 +11,11 @@ "Brokers": { "NTG": { "Mock": true, - "Cookie": "" + "Token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIzMTAyOSIsInVuaXF1ZV9uYW1lIjoibm9lYXN5cnVuc3RydWNraW5nQGdtYWlsLmNvbSIsImp0aSI6IjFmYzhmNjk1LTQzNTYtNGYzZS05NWY1LWZkNWVjMDJlMDkxNyIsImlhdCI6IjEvMTAvMjAyMiAxMTo1OTozNiBQTSIsIm50Z3ZSb2xlIjoiQ2FycmllckFwcHJvdmVkIiwidXNlckNhcnJpZXJzIjoiMTUzNDIzIiwib3RyVXNlciI6IkZhbHNlIiwibmJmIjoxNjQxODU5MTc2LCJleHAiOjE2NDE5NDE5NzYsImlzcyI6Ik5URyBTZWN1cml0eSBUb2tlbiBTZXJ2aWNlIiwiYXVkIjoiTlRHIn0.kpHOBTtcQhbdloAw7xEjnkzfxf4ToMgidrLCMomZmnmKQHlD_7OQuI8nQiCTHc_ntuGtt8Ui92kwWWUiLwN_2tT2vC7Jy6m9IjwqgbAzsgTLi4jAbIwITD-awiDh4FUKDwGq3XpEjs-i7XM3rI7tTk7jg9QSDId8EF3Pt5fJq6QhztC6y7-JaSFQZLMtkSCAWmOQx_TgKgVoVbgMeiqhHbZ2hhoA7TtpEIIL5Gnfq46t3E18ExdrsO96ZCGQGcBw5x8J1ustq2cwdlFKeg4ULNWAAd1ay1hojRa7jCHs98AcoJ3Nts9-o7yEMuN2rrfpK_nm68nciwFtF-ke1KoiBg", + "Username": "noeasyrunstrucking@gmail.com", + "Password": "thumper123" } }, - "EmailerEnabled": false, "Emailer": { "From": "breellocaldev@gmail.com", "SMTP": "", @@ -22,5 +23,6 @@ "IMAP": "imap.gmail.com:993", "Password": "gojfkkfrkmtxzyro", "Limit": 0 - } + }, + "EmailerEnabled": false } \ No newline at end of file diff --git a/config/config.go b/config/config.go index 5068fc0..a896e59 100644 --- a/config/config.go +++ b/config/config.go @@ -17,8 +17,10 @@ type Config struct { Once bool Brokers struct { NTG struct { - Mock bool - Cookie string + Mock bool + Token string + Username string + Password string } } Emailer contact.Emailer diff --git a/testdata/scrape_cookies.sh b/testdata/scrape_cookies.sh index 2ea72a1..d44915b 100644 --- a/testdata/scrape_cookies.sh +++ b/testdata/scrape_cookies.sh @@ -4,7 +4,8 @@ main() { set -e set -o pipefail - local cookie_original="/Users/breel/Library/Application Support/Firefox/Profiles/rek8wrhm.default-release/cookies.sqlite" + local profile_root="/Users/breel/Library/Application Support/Firefox/Profiles/nk8e5y5u.ronnie" + local cookie_original="${profile_root%/}/cookies.sqlite" f=$(mktemp) cleanup() ( rm -f "$f"; ) trap cleanup EXIT