update config for fastexact, gzip decompress fastexact at parse time

broker/fastexact.go

@@ -2,12 +2,14 @@ package broker
 
 import (
 	"bytes"
+	"compress/gzip"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"local/storage"
 	"local/truckstop/config"
+	"local/truckstop/logtr"
 	"net/http"
 	"strconv"
 	"strings"
@@ -104,6 +106,7 @@ func (fe FastExact) searchOne(state config.State) ([]Job, error) {
 		return nil, err
 	}
 	resp, err := fe.doer.doRequest(req)
+	logtr.Verbosef("req: %+v => resp: %+v", req, resp)
 	if err != nil {
 		return nil, err
 	}
@@ -132,6 +135,16 @@ func (fe FastExact) doRequest(req *http.Request) (*http.Response, error) {
 }
 
 func (fe FastExact) parse(resp *http.Response) ([]Job, error) {
+	b, _ := ioutil.ReadAll(resp.Body)
+	if !bytes.HasPrefix(bytes.TrimSpace(b), []byte("<")) {
+		gzip, err := gzip.NewReader(bytes.NewReader(b))
+		if err != nil {
+			return nil, err
+		}
+		b, _ = ioutil.ReadAll(gzip)
+	}
+	logtr.Verbosef("fe.parse %s", b)
+	resp.Body = io.NopCloser(bytes.NewReader(b))
 	doc, err := goquery.NewDocumentFromReader(resp.Body)
 	if err != nil {
 		return nil, err
@@ -174,6 +187,8 @@ func (fe FastExact) parse(resp *http.Response) ([]Job, error) {
 
 		job.Meta = fmt.Sprintf(`dimensions:%s`, strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(columns[5], " ", ""), "\n", "")))
 
+		logtr.Verbosef("fe.parse %+v => %+v", columns, job)
+
 		result = append(result, job)
 	})
 	return result, nil

broker/fastexact_test.go

@@ -3,11 +3,47 @@ package broker
 import (
 	"local/storage"
 	"local/truckstop/config"
+	"os"
+	"path"
 	"testing"
 
 	"golang.org/x/time/rate"
 )
 
+func TestFastExactReal(t *testing.T) {
+	if os.Getenv("INTEGRATION_FAST_EXACT") == "" {
+		t.Skip("$INTEGRATION_FAST_EXACT not set")
+	}
+	os.Setenv("CONFIG", "../config.json")
+	if err := config.Refresh(nil); err != nil {
+		t.Fatal(err)
+	}
+	conf := config.Get()
+	conf.Storage = []string{"files", path.Join(t.TempDir(), "storage")}
+	os.Setenv("CONFIG", path.Join(t.TempDir(), "config.json"))
+	config.Set(*conf)
+	if err := config.Refresh(nil); err != nil {
+		t.Fatal(err)
+	}
+	states := []config.State{
+		config.State("NC"),
+	}
+
+	fe := NewFastExact()
+	if err := fe.login(); err != nil {
+		t.Fatal(err)
+	}
+	jobs, err := fe.search(states)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("%d", len(jobs))
+	for i := range jobs {
+		t.Logf("[%d] %+v", i, jobs[i])
+	}
+}
+
 func TestFastExactLogin(t *testing.T) {
 	limiter = rate.NewLimiter(rate.Limit(60.0), 1)
 	fe := NewFastExact().WithMock()

config.json

@@ -71,6 +71,11 @@
    },
    "Once": true,
    "Brokers": {
+      "FastExact": {
+         "Mock": false,
+         "Username": "birdman",
+         "Password": "166647"
+      },
       "NTG": {
          "JobInfo": true,
          "Mock": true,

config.main_test.json

@@ -71,6 +71,11 @@
    },
    "Once": true,
    "Brokers": {
+      "FastExact": {
+         "Mock": true,
+         "Username": "u",
+         "Password": "p"
+      },
       "NTG": {
          "JobInfo": true,
          "Mock": true,