package broker import ( "bytes" "encoding/json" "errors" "fmt" "io" "io/ioutil" "local/truckstop/config" "local/truckstop/logtr" "local/truckstop/zip" "net/http" "time" ) type NTGVision struct { searcher interface { searchStates(states []config.State) (io.ReadCloser, error) searchJobReadCloser(id int64) (io.ReadCloser, error) } } var ErrNoAuth = errors.New("not authorized") type ntgVisionJob struct { ID int64 `json:"id"` PickupDate string `json:"sDate"` PickupCity string `json:"sCity"` PickupState string `json:"sState"` DropoffDate string `json:"cDate"` DropoffCity string `json:"cCity"` DropoffState string `json:"cState"` Miles int `json:"miles"` Weight int `json:"weight"` Equipment string `json:"equip"` Temp string `json:"temp"` jobinfo ntgVisionJobInfo } type ntgVisionJobInfo struct { StopsInfo []struct { StopHours string `json:"stopHours"` AppointmentTime string `json:"appointmentTime"` Instructions string `json:"instructions"` IsDropTrailer bool `json:"isDropTrailer"` } `json:"stopinfos"` PayUpTo float32 `json:"payUpTo"` TotalCarrierRate float32 `json:"totalCarrierRate"` LoadState string `json:"loadStatus"` //CanBidNow bool `json:"canBidNow"` } func (ji ntgVisionJobInfo) IsZero() bool { return len(ji.StopsInfo) == 0 && ji.TotalCarrierRate == 0 && ji.PayUpTo == 0 && ji.LoadState == "" } func (ji ntgVisionJobInfo) String() string { if ji.IsZero() { return "" } out := "" if ji.PayUpTo != 0 { out = fmt.Sprintf("\nPAYS:%v\n%s", ji.PayUpTo, out) } if ji.TotalCarrierRate != 0 { out = fmt.Sprintf("%s Total_Carrier_Rate:%v", out, ji.TotalCarrierRate) } if ji.LoadState != "" { out = fmt.Sprintf("%s Auction:%s", out, ji.LoadState) } if len(ji.StopsInfo) != 2 { return out } return fmt.Sprintf( "%s Pickup:{Hours:%s Notes:%s, DropTrailer:%v} Dropoff:{Appointment:%s Notes:%s}", out, ji.StopsInfo[0].StopHours, ji.StopsInfo[0].Instructions, ji.StopsInfo[0].IsDropTrailer, ji.StopsInfo[1].AppointmentTime, ji.StopsInfo[1].Instructions, ) } func (ntgJob *ntgVisionJob) JobInfo() (ntgVisionJobInfo, error) { if !config.Get().Brokers.NTG.JobInfo { return ntgJob.jobinfo, nil } if !ntgJob.jobinfo.IsZero() { return ntgJob.jobinfo, nil } db := config.Get().DB() key := fmt.Sprintf("ntg_job_info_%v", ntgJob.ID) if b, err := db.Get(key); err != nil { } else if err := json.Unmarshal(b, &ntgJob.jobinfo); err == nil { return ntgJob.jobinfo, nil } ntg := NewNTGVision() ji, err := ntg.searchJob(ntgJob.ID) if err == nil { ntgJob.jobinfo = ji b, err := json.Marshal(ntgJob.jobinfo) if err == nil { db.Set(key, b) } } return ji, err } func (ntg NTGVision) searchJobReadCloser(id int64) (io.ReadCloser, error) { time.Sleep(config.Get().Interval.JobInfo.Get()) request, err := http.NewRequest(http.MethodGet, fmt.Sprintf(config.Get().Brokers.NTG.LoadPageAPIURIFormat, id), nil) if err != nil { return nil, err } setNTGHeaders(request) resp, err := do(request) if err != nil { return nil, err } defer resp.Body.Close() b, _ := ioutil.ReadAll(resp.Body) logtr.Debugf("fetch ntg job info %+v: %d: %s", request, resp.StatusCode, b) return io.NopCloser(bytes.NewReader(b)), nil } func (ntgJob *ntgVisionJob) Job() Job { pickup, _ := time.ParseInLocation("01/02/06", ntgJob.PickupDate, time.Local) dropoff, _ := time.ParseInLocation("01/02/06", ntgJob.DropoffDate, time.Local) return Job{ ID: fmt.Sprintf("ntg-%d", ntgJob.ID), URI: fmt.Sprintf(config.Get().Brokers.NTG.LoadPageURIFormat, ntgJob.ID), Pickup: JobLocation{ Date: pickup, City: ntgJob.PickupCity, State: ntgJob.PickupState, }, Dropoff: JobLocation{ Date: dropoff, City: ntgJob.DropoffCity, State: ntgJob.DropoffState, }, Miles: ntgJob.Miles, Weight: ntgJob.Weight, Meta: fmt.Sprintf("equipment:%s", ntgJob.Equipment), secrets: func(j *Job) { jobInfo, err := ntgJob.JobInfo() if err != nil { logtr.Errorf("failed to get jobinfo: %v", err) return } j.Meta = jobInfo.String() j.Pays = fmt.Sprint(jobInfo.PayUpTo) }, } } func NewNTGVision() NTGVision { ntgv := NTGVision{} ntgv.searcher = ntgv if config.Get().Brokers.NTG.Mock { ntgv = ntgv.WithMock() } return ntgv } func (ntg NTGVision) WithMock() NTGVision { ntg.searcher = NewNTGVisionMock() return ntg } func (ntg NTGVision) searchJob(id int64) (ntgVisionJobInfo, error) { rc, err := ntg.searcher.searchJobReadCloser(id) if err != nil { return ntgVisionJobInfo{}, err } defer rc.Close() b, err := ioutil.ReadAll(rc) if err != nil { return ntgVisionJobInfo{}, fmt.Errorf("failed to readall search job result: %w", err) } var result ntgVisionJobInfo err = json.Unmarshal(b, &result) if err != nil { err = fmt.Errorf("failed to parse job info: %w: %s", err, b) } return result, err } func (ntg NTGVision) SearchZips(zips []string) ([]Job, error) { radius := config.Get().Brokers.RadiusMiles statesm := map[string]struct{}{} for _, z := range zips { somestates := zip.GetStatesWithin(zip.Get(z), radius) for _, state := range somestates { statesm[state] = struct{}{} } } states := make([]config.State, 0, len(statesm)) for state := range statesm { states = append(states, config.State(state)) } if len(states) == 0 { return nil, fmt.Errorf("failed to map zipcodes %+v to any states", zips) } jobs, err := ntg.SearchStates(states) if err != nil { return nil, err } for i := len(jobs) - 1; i >= 0; i-- { ok := false for _, z := range zips { ok = ok || zip.Get(z).MilesTo(zip.FromCityState(jobs[i].Pickup.City, jobs[i].Pickup.State)) <= radius } if !ok { jobs[i], jobs[len(jobs)-1] = jobs[len(jobs)-1], jobs[i] jobs = jobs[:len(jobs)-1] } } return jobs, nil } func (ntg NTGVision) workingHours(now time.Time) bool { // TODO assert M-F 9-4 EST now = now.In(time.FixedZone("EST", -5*60*60)) working := config.Get().Brokers.NTG.Working logtr.Debugf("ntg.workingHours: now=%s, weekday=%v, hour=%v (ok=%+v)", now.String(), now.Weekday(), now.Hour(), working) if ok := func() bool { for _, hr := range working.Hours { if now.Hour() == hr { return true } } return false }(); !ok { return false } if ok := func() bool { for _, weekday := range working.Weekdays { if now.Weekday() == time.Weekday(weekday) { return true } } return false }(); !ok { return false } return true } func (ntg NTGVision) SearchStates(states []config.State) ([]Job, error) { if !ntg.workingHours(time.Now()) { lastNtgB, _ := config.Get().DB().Get("ntg_last_search_states") var jobs []Job json.Unmarshal(lastNtgB, &jobs) logtr.Verbosef("ntg.SearchStates: outside of working hours so returning ntg_last_search_states: %+v", jobs) return jobs, nil } rc, err := ntg.searcher.searchStates(states) if err != nil { return nil, err } defer rc.Close() b, err := ioutil.ReadAll(rc) if err != nil { return nil, err } logtr.Debugf("ntg search for %+v: %s", states, b) var ntgjobs []ntgVisionJob err = json.Unmarshal(b, &ntgjobs) if err != nil { return nil, err } jobs := make([]Job, len(ntgjobs)) for i := range jobs { jobs[i] = ntgjobs[i].Job() } jobsB, err := json.Marshal(jobs) if err == nil { config.Get().DB().Set("ntg_last_search_states", jobsB) logtr.Verbosef("ntg.SearchStates: in working hours so setting ntg_last_search_states: %+v", jobs) } return jobs, nil } func getNTGTokenKey() string { return "brokers_ntg_token" } func getNTGToken() string { db := config.Get().DB() b, _ := db.Get(getNTGTokenKey()) return string(b) } func setNTGToken(token string) { db := config.Get().DB() db.Set(getNTGTokenKey(), []byte(token)) } func (ntg NTGVision) searchStates(states []config.State) (io.ReadCloser, error) { if getNTGToken() == "" { logtr.Debugf("NTG token is empty, refreshing ntg auth") if err := ntg.refreshAuth(); err != nil { return nil, err } } rc, err := ntg._searchStates(states) if err == ErrNoAuth { logtr.Debugf("err no auth on search, refreshing ntg auth") if err := ntg.refreshAuth(); err != nil { return nil, err } rc, err = ntg._searchStates(states) } return rc, err } func (ntg NTGVision) refreshAuth() error { err := ntg._refreshAuth() if err != nil { logtr.SOSf("failed to refresh ntg auth: %v", err) } return err } func (ntg NTGVision) _refreshAuth() error { logtr.Infof("refreshing ntg auth...") b, _ := json.Marshal(map[string]string{ "username": config.Get().Brokers.NTG.Username, "password": config.Get().Brokers.NTG.Password, }) request, err := http.NewRequest(http.MethodPost, "https://ntgvision.com/api/v1/sts/Login", bytes.NewReader(b)) if err != nil { return err } setNTGHeaders(request) resp, err := do(request) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { b, _ := ioutil.ReadAll(resp.Body) return fmt.Errorf("failed refreshing token: (%d): %s", resp.StatusCode, b) } var v struct { Token string `json:"token"` } if err := json.NewDecoder(resp.Body).Decode(&v); err != nil { return err } if len(v.Token) == 0 { return errors.New("failed to get token from login call") } setNTGToken(v.Token) return nil } func (ntg NTGVision) _searchStates(states []config.State) (io.ReadCloser, error) { request, err := ntg.newRequest(states) if err != nil { return nil, err } resp, err := do(request) if err != nil { return nil, err } defer resp.Body.Close() b, _ := ioutil.ReadAll(resp.Body) resp.Body = io.NopCloser(bytes.NewReader(b)) if resp.StatusCode != http.StatusOK { b, _ := ioutil.ReadAll(resp.Body) resp.Body.Close() request2, _ := ntg.newRequest(states) requestb, _ := ioutil.ReadAll(request2.Body) logtr.Debugf("ntg auth bad status: url=%s, status=%v, body=%s, headers=%+v, request=%+v, requestb=%s", request.URL.String(), resp.StatusCode, b, resp.Header, request2, requestb) if resp.StatusCode > 400 && resp.StatusCode < 404 || resp.StatusCode == 417 { // TODO wtf is 417 for logtr.Debugf("ntg auth bad status: err no auth") return nil, ErrNoAuth } return nil, fmt.Errorf("bad status searching ntg: %d: %s", resp.StatusCode, b) } return resp.Body, nil } func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) { body, err := json.Marshal(map[string]interface{}{ "OriginFromDate": time.Now().Add(time.Hour * -24).UTC().Format("2006-01-02T15:04:05.000Z"), "OriginToDate": time.Now().UTC().Add(time.Hour * 24 * 30).Format("2006-01-02T15:04:05.000Z"), "DestinationFromDate": nil, "DestinationToDate": nil, "OriginState": states, "OriginRadius": 100, "DestinationState": []string{}, "DestinationRadius": 100, "EquipmentTypes": []string{"STRAIGHT TRUCK", "str truck w/ lift gate"}, "MaxMilesLimit": nil, "MaxNumberOfStopsLimit": nil, "MaxWeightLimit": nil, "MinMilesLimit": nil, "SavedSearchId": 2956, }) if err != nil { return nil, err } request, err := http.NewRequest(http.MethodPost, "https://ntgvision.com/api/v1/load/LoadBoardSearchResults", bytes.NewReader(body)) if err != nil { return nil, err } setNTGHeaders(request) request.Header.Set("Authorization", "Bearer "+getNTGToken()) request.Header.Set("Cookie", "NTGAuthToken="+getNTGToken()) return request, nil } func setNTGHeaders(request *http.Request) { request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0") request.Header.Set("Accept", "application/json, text/plain, */*") request.Header.Set("Accept-Language", "en-US,en;q=0.5") request.Header.Set("Accept-Encoding", "gzip, deflate, br") request.Header.Set("Content-Type", "application/json;charset=utf-8") //request.Header.Set("Authorization", "Bearer "+getNTGToken()) request.Header.Set("Origin", "https://ntgvision.com") request.Header.Set("DNT", "1") request.Header.Set("Connection", "keep-alive") //request.Header.Set("Cookie", config.Get().Brokers.NTG.Cookie) request.Header.Set("Sec-Fetch-Dest", "empty") request.Header.Set("Sec-Fetch-Mode", "cors") request.Header.Set("Sec-Fetch-Site", "same-origin") request.Header.Set("Pragma", "no-cache") request.Header.Set("Cache-Control", "no-cache") }