update config for fastexact, gzip decompress fastexact at parse time

broker/fastexact.go

@@ -2,12 +2,14 @@ package broker
 
 import (
 	"bytes"
+	"compress/gzip"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"local/storage"
 	"local/truckstop/config"
+	"local/truckstop/logtr"
 	"net/http"
 	"strconv"
 	"strings"
@@ -104,6 +106,7 @@ func (fe FastExact) searchOne(state config.State) ([]Job, error) {
 		return nil, err
 	}
 	resp, err := fe.doer.doRequest(req)
+	logtr.Verbosef("req: %+v => resp: %+v", req, resp)
 	if err != nil {
 		return nil, err
 	}
@@ -132,6 +135,16 @@ func (fe FastExact) doRequest(req *http.Request) (*http.Response, error) {
 }
 
 func (fe FastExact) parse(resp *http.Response) ([]Job, error) {
+	b, _ := ioutil.ReadAll(resp.Body)
+	if !bytes.HasPrefix(bytes.TrimSpace(b), []byte("<")) {
+		gzip, err := gzip.NewReader(bytes.NewReader(b))
+		if err != nil {
+			return nil, err
+		}
+		b, _ = ioutil.ReadAll(gzip)
+	}
+	logtr.Verbosef("fe.parse %s", b)
+	resp.Body = io.NopCloser(bytes.NewReader(b))
 	doc, err := goquery.NewDocumentFromReader(resp.Body)
 	if err != nil {
 		return nil, err
@@ -174,6 +187,8 @@ func (fe FastExact) parse(resp *http.Response) ([]Job, error) {
 
 		job.Meta = fmt.Sprintf(`dimensions:%s`, strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(columns[5], " ", ""), "\n", "")))
 
+		logtr.Verbosef("fe.parse %+v => %+v", columns, job)
+
 		result = append(result, job)
 	})
 	return result, nil