truckstop/broker/ntgvision.go

package broker

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"local/truckstop/config"
	"local/truckstop/logtr"
	"local/truckstop/zip"
	"net/http"
	"time"
)

type NTGVision struct {
	searcher interface {
		searchStates(states []config.State) (io.ReadCloser, error)
		searchJobReadCloser(id int64) (io.ReadCloser, error)
	}
}

var ErrNoAuth = errors.New("not authorized")

type ntgVisionJob struct {
	ID           int64  `json:"id"`
	PickupDate   string `json:"sDate"`
	PickupCity   string `json:"sCity"`
	PickupState  string `json:"sState"`
	DropoffDate  string `json:"cDate"`
	DropoffCity  string `json:"cCity"`
	DropoffState string `json:"cState"`
	Miles        int    `json:"miles"`
	Weight       int    `json:"weight"`
	Equipment    string `json:"equip"`
	Temp         string `json:"temp"`
	jobinfo      ntgVisionJobInfo
}

type ntgVisionJobInfo struct {
	StopsInfo []struct {
		StopHours       string `json:"stopHours"`
		AppointmentTime string `json:"appointmentTime"`
		Instructions    string `json:"instructions"`
		IsDropTrailer   bool   `json:"isDropTrailer"`
	} `json:"stopinfos"`
	PayUpTo          float32 `json:"payUpTo"`
	TotalCarrierRate float32 `json:"totalCarrierRate"`
	LoadState        string  `json:"loadStatus"`
	//CanBidNow bool    `json:"canBidNow"`
}

func (ji ntgVisionJobInfo) IsZero() bool {
	return len(ji.StopsInfo) == 0 && ji.TotalCarrierRate == 0 && ji.PayUpTo == 0 && ji.LoadState == ""
}

func (ji ntgVisionJobInfo) String() string {
	if ji.IsZero() {
		return ""
	}
	out := ""
	if ji.PayUpTo != 0 {
		out = fmt.Sprintf("\nPAYS:%v\n%s", ji.PayUpTo, out)
	}
	if ji.TotalCarrierRate != 0 {
		out = fmt.Sprintf("%s Total_Carrier_Rate:%v", out, ji.TotalCarrierRate)
	}
	if ji.LoadState != "" {
		out = fmt.Sprintf("%s Auction:%s", out, ji.LoadState)
	}
	if len(ji.StopsInfo) != 2 {
		return out
	}
	return fmt.Sprintf(
		"%s Pickup:{Hours:%s Notes:%s, DropTrailer:%v} Dropoff:{Appointment:%s Notes:%s}",
		out,
		ji.StopsInfo[0].StopHours,
		ji.StopsInfo[0].Instructions,
		ji.StopsInfo[0].IsDropTrailer,
		ji.StopsInfo[1].AppointmentTime,
		ji.StopsInfo[1].Instructions,
	)
}

func (ntgJob *ntgVisionJob) JobInfo() (ntgVisionJobInfo, error) {
	if !config.Get().Brokers.NTG.JobInfo {
		return ntgJob.jobinfo, nil
	}
	if !ntgJob.jobinfo.IsZero() {
		return ntgJob.jobinfo, nil
	}
	db := config.Get().DB()
	key := fmt.Sprintf("ntg_job_info_%v", ntgJob.ID)
	if b, err := db.Get(key); err != nil {
	} else if err := json.Unmarshal(b, &ntgJob.jobinfo); err == nil {
		return ntgJob.jobinfo, nil
	}
	ntg := NewNTGVision()
	ji, err := ntg.searchJob(ntgJob.ID)
	if err == nil {
		ntgJob.jobinfo = ji
		b, err := json.Marshal(ntgJob.jobinfo)
		if err == nil {
			db.Set(key, b)
		}
	}
	return ji, err
}

func (ntg NTGVision) searchJobReadCloser(id int64) (io.ReadCloser, error) {
	time.Sleep(config.Get().Interval.JobInfo.Get())
	request, err := http.NewRequest(http.MethodGet, fmt.Sprintf(config.Get().Brokers.NTG.LoadPageAPIURIFormat, id), nil)
	if err != nil {
		return nil, err
	}
	setNTGHeaders(request)
	resp, err := do(request)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	b, _ := ioutil.ReadAll(resp.Body)
	logtr.Debugf("fetch ntg job info %+v: %d: %s", request, resp.StatusCode, b)
	return io.NopCloser(bytes.NewReader(b)), nil
}

func (ntgJob *ntgVisionJob) Job() Job {
	pickup, _ := time.ParseInLocation("01/02/06", ntgJob.PickupDate, time.Local)
	dropoff, _ := time.ParseInLocation("01/02/06", ntgJob.DropoffDate, time.Local)
	return Job{
		ID:  fmt.Sprintf("ntg-%d", ntgJob.ID),
		URI: fmt.Sprintf(config.Get().Brokers.NTG.LoadPageURIFormat, ntgJob.ID),
		Pickup: JobLocation{
			Date:  pickup,
			City:  ntgJob.PickupCity,
			State: ntgJob.PickupState,
		},
		Dropoff: JobLocation{
			Date:  dropoff,
			City:  ntgJob.DropoffCity,
			State: ntgJob.DropoffState,
		},
		Miles:  ntgJob.Miles,
		Weight: ntgJob.Weight,
		Meta:   fmt.Sprintf("equipment:%s", ntgJob.Equipment),
		secrets: func(j *Job) {
			jobInfo, err := ntgJob.JobInfo()
			if err != nil {
				logtr.Errorf("failed to get jobinfo: %v", err)
				return
			}
			j.Meta = jobInfo.String()
			j.Pays = fmt.Sprint(jobInfo.PayUpTo)
		},
	}
}

func NewNTGVision() NTGVision {
	ntgv := NTGVision{}
	ntgv.searcher = ntgv
	if config.Get().Brokers.NTG.Mock {
		ntgv = ntgv.WithMock()
	}
	return ntgv
}

func (ntg NTGVision) WithMock() NTGVision {
	ntg.searcher = NewNTGVisionMock()
	return ntg
}

func (ntg NTGVision) searchJob(id int64) (ntgVisionJobInfo, error) {
	rc, err := ntg.searcher.searchJobReadCloser(id)
	if err != nil {
		return ntgVisionJobInfo{}, err
	}
	defer rc.Close()
	b, err := ioutil.ReadAll(rc)
	if err != nil {
		return ntgVisionJobInfo{}, fmt.Errorf("failed to readall search job result: %w", err)
	}
	var result ntgVisionJobInfo
	err = json.Unmarshal(b, &result)
	if err != nil {
		err = fmt.Errorf("failed to parse job info: %w: %s", err, b)
	}
	return result, err
}

func (ntg NTGVision) SearchZips(zips []string) ([]Job, error) {
	radius := config.Get().Brokers.RadiusMiles
	statesm := map[string]struct{}{}
	for _, z := range zips {
		somestates := zip.GetStatesWithin(zip.Get(z), radius)
		for _, state := range somestates {
			statesm[state] = struct{}{}
		}
	}
	states := make([]config.State, 0, len(statesm))
	for state := range statesm {
		states = append(states, config.State(state))
	}
	if len(states) == 0 {
		return nil, fmt.Errorf("failed to map zipcodes %+v to any states", zips)
	}
	jobs, err := ntg.SearchStates(states)
	if err != nil {
		return nil, err
	}
	for i := len(jobs) - 1; i >= 0; i-- {
		ok := false
		for _, z := range zips {
			ok = ok || zip.Get(z).MilesTo(zip.FromCityState(jobs[i].Pickup.City, jobs[i].Pickup.State)) <= radius
		}
		if !ok {
			jobs[i], jobs[len(jobs)-1] = jobs[len(jobs)-1], jobs[i]
			jobs = jobs[:len(jobs)-1]
		}
	}
	return jobs, nil
}

func (ntg NTGVision) workingHours(now time.Time) bool {
	// TODO assert M-F 9-4 EST
	now = now.In(time.FixedZone("EST", -5*60*60))
	switch now.Weekday() {
	case time.Sunday, time.Saturday:
		return false
	}
	switch now.Hour() {
	case 9, 10, 11, 12, 13, 14, 15, 16:
	default:
		return false
	}
	return true
}

func (ntg NTGVision) SearchStates(states []config.State) ([]Job, error) {
	if ntg.workingHours(time.Now()) {
		return nil, nil
	}

	rc, err := ntg.searcher.searchStates(states)
	if err != nil {
		return nil, err
	}
	defer rc.Close()

	b, err := ioutil.ReadAll(rc)
	if err != nil {
		return nil, err
	}

	logtr.Debugf("ntg search for %+v: %s", states, b)

	var ntgjobs []ntgVisionJob
	err = json.Unmarshal(b, &ntgjobs)

	jobs := make([]Job, len(ntgjobs))
	for i := range jobs {
		jobs[i] = ntgjobs[i].Job()
	}
	return jobs, err
}

func getNTGTokenKey() string {
	return "brokers_ntg_token"
}

func getNTGToken() string {
	db := config.Get().DB()
	b, _ := db.Get(getNTGTokenKey())
	return string(b)
}

func setNTGToken(token string) {
	db := config.Get().DB()
	db.Set(getNTGTokenKey(), []byte(token))
}

func (ntg NTGVision) searchStates(states []config.State) (io.ReadCloser, error) {
	if getNTGToken() == "" {
		logtr.Debugf("NTG token is empty, refreshing ntg auth")
		if err := ntg.refreshAuth(); err != nil {
			return nil, err
		}
	}
	rc, err := ntg._searchStates(states)
	if err == ErrNoAuth {
		logtr.Debugf("err no auth on search, refreshing ntg auth")
		if err := ntg.refreshAuth(); err != nil {
			return nil, err
		}
		rc, err = ntg._searchStates(states)
	}
	return rc, err
}

func (ntg NTGVision) refreshAuth() error {
	err := ntg._refreshAuth()
	if err != nil {
		logtr.SOSf("failed to refresh ntg auth: %v", err)
	}
	return err
}

func (ntg NTGVision) _refreshAuth() error {
	logtr.Infof("refreshing ntg auth...")
	b, _ := json.Marshal(map[string]string{
		"username": config.Get().Brokers.NTG.Username,
		"password": config.Get().Brokers.NTG.Password,
	})
	request, err := http.NewRequest(http.MethodPost, "https://ntgvision.com/api/v1/sts/Login", bytes.NewReader(b))
	if err != nil {
		return err
	}
	setNTGHeaders(request)
	resp, err := do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		b, _ := ioutil.ReadAll(resp.Body)
		return fmt.Errorf("failed refreshing token: (%d): %s", resp.StatusCode, b)
	}
	var v struct {
		Token string `json:"token"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
		return err
	}
	if len(v.Token) == 0 {
		return errors.New("failed to get token from login call")
	}
	setNTGToken(v.Token)
	return nil
}

func (ntg NTGVision) _searchStates(states []config.State) (io.ReadCloser, error) {
	request, err := ntg.newRequest(states)
	if err != nil {
		return nil, err
	}
	resp, err := do(request)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		b, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		request2, _ := ntg.newRequest(states)
		requestb, _ := ioutil.ReadAll(request2.Body)
		logtr.Debugf("ntg auth bad status: url=%s, status=%v, body=%s, headers=%+v, request=%+v, requestb=%s", request.URL.String(), resp.StatusCode, b, resp.Header, request2, requestb)
		if resp.StatusCode > 400 && resp.StatusCode < 404 || resp.StatusCode == 417 { // TODO wtf is 417 for
			logtr.Debugf("ntg auth bad status: err no auth")
			return nil, ErrNoAuth
		}
		return nil, fmt.Errorf("bad status searching ntg: %d: %s", resp.StatusCode, b)
	}
	return resp.Body, nil
}

func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) {
	body, err := json.Marshal(map[string]interface{}{
		"OriginFromDate":        time.Now().Add(time.Hour * -24).UTC().Format("2006-01-02T15:04:05.000Z"),
		"OriginToDate":          time.Now().UTC().Add(time.Hour * 24 * 30).Format("2006-01-02T15:04:05.000Z"),
		"DestinationFromDate":   nil,
		"DestinationToDate":     nil,
		"OriginState":           states,
		"OriginRadius":          100,
		"DestinationState":      []string{},
		"DestinationRadius":     100,
		"EquipmentTypes":        []string{"STRAIGHT TRUCK", "str truck w/ lift gate"},
		"MaxMilesLimit":         nil,
		"MaxNumberOfStopsLimit": nil,
		"MaxWeightLimit":        nil,
		"MinMilesLimit":         nil,
		"SavedSearchId":         2956,
	})
	if err != nil {
		return nil, err
	}
	request, err := http.NewRequest(http.MethodPost, "https://ntgvision.com/api/v1/load/LoadBoardSearchResults", bytes.NewReader(body))
	if err != nil {
		return nil, err
	}
	setNTGHeaders(request)
	request.Header.Set("Authorization", "Bearer "+getNTGToken())
	request.Header.Set("Cookie", "NTGAuthToken="+getNTGToken())

	return request, nil
}

func setNTGHeaders(request *http.Request) {
	request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0")
	request.Header.Set("Accept", "application/json, text/plain, */*")
	request.Header.Set("Accept-Language", "en-US,en;q=0.5")
	request.Header.Set("Accept-Encoding", "gzip, deflate, br")
	request.Header.Set("Content-Type", "application/json;charset=utf-8")
	//request.Header.Set("Authorization", "Bearer "+getNTGToken())
	request.Header.Set("Origin", "https://ntgvision.com")
	request.Header.Set("DNT", "1")
	request.Header.Set("Connection", "keep-alive")
	//request.Header.Set("Cookie", config.Get().Brokers.NTG.Cookie)
	request.Header.Set("Sec-Fetch-Dest", "empty")
	request.Header.Set("Sec-Fetch-Mode", "cors")
	request.Header.Set("Sec-Fetch-Site", "same-origin")
	request.Header.Set("Pragma", "no-cache")
	request.Header.Set("Cache-Control", "no-cache")
}