package broker import ( "context" "encoding/json" "errors" "local/storage" "local/truckstop/config" "local/truckstop/logtr" "net/http" "net/http/cookiejar" "strings" "time" "golang.org/x/time/rate" ) // once per minute var authlimiter = rate.NewLimiter(rate.Limit(1.0/60.0), 1) // thrice per minute var limiter = rate.NewLimiter(rate.Limit(1.0/20.0), 1) type Broker interface { Search([]config.State) ([]Job, error) } func do(r *http.Request) (*http.Response, error) { return _do(config.Get().DB(), r) } func _do(db storage.DB, r *http.Request) (*http.Response, error) { limiter.Wait(context.Background()) if strings.Contains(strings.ToLower(r.URL.Path), "login") { authlimiter.Wait(context.Background()) } client := &http.Client{ Timeout: time.Hour, } newjar, err := cookiejar.New(&cookiejar.Options{}) if err != nil { return nil, errors.New("failed to make a cookie jar: " + err.Error()) } client.Jar = newjar cookieJarKey := "cookies_" + r.URL.Host cookies, err := getCookies(db, cookieJarKey) if err != nil { logtr.Errorf("failed to load cookies: %v", err) } else { client.Jar.SetCookies(r.URL, cookies) } resp, err := client.Do(r) if err != nil { return nil, err } if err := setCookies(db, cookieJarKey, client.Jar.Cookies(r.URL)); err != nil { logtr.Errorf("failed to set cookies: %v", err) } return resp, err } func getCookies(db storage.DB, host string) ([]*http.Cookie, error) { b, err := db.Get(host) if err != nil { return nil, err } var result []*http.Cookie err = json.Unmarshal(b, &result) return result, err } func setCookies(db storage.DB, host string, cookies []*http.Cookie) error { b, err := json.Marshal(cookies) if err != nil { return err } return db.Set(host, b) }