Create fetcher for http requests, processing, and saving

fetch/fetch.go

@@ -0,0 +1,53 @@
+package fetch
+
+import (
+	"io/ioutil"
+	"net/http"
+)
+
+type Savable interface {
+	Namespace() string
+	Key() string
+	Value() []byte
+}
+
+type Fetch struct {
+	process func([]byte) ([]Savable, error)
+	save    func(string, string, []byte) error
+	client  *http.Client
+}
+
+func New(process func([]byte) ([]Savable, error), save func(string, string, []byte) error) (*Fetch, error) {
+	return &Fetch{
+		save:    save,
+		process: process,
+		client:  &http.Client{},
+	}, nil
+}
+
+func (fetcher *Fetch) FetchProcessSave(url string) error {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := fetcher.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	items, err := fetcher.process(b)
+	if err != nil {
+		return err
+	}
+	for i := range items {
+		err := fetcher.save(items[i].Namespace(), items[i].Key(), items[i].Value())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

fetch/fetch_test.go

@@ -0,0 +1,88 @@
+package fetch
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+type mockSavable struct {
+	ns string
+	k  string
+	v  []byte
+}
+
+func (ms *mockSavable) Namespace() string { return ms.ns }
+func (ms *mockSavable) Key() string       { return ms.k }
+func (ms *mockSavable) Value() []byte     { return ms.v }
+
+func Test_Fetch(t *testing.T) {
+	s := mockRemote()
+	defer s.Close()
+	cases := []struct {
+		process func([]byte) ([]Savable, error)
+		save    func(string, string, []byte) error
+		err     error
+	}{
+		{
+			process: func([]byte) ([]Savable, error) {
+				return []Savable{&mockSavable{
+					ns: "hello",
+					k:  "world",
+					v:  []byte("!"),
+				}}, nil
+			},
+			save: func(string, string, []byte) error {
+				return nil
+			},
+			err: nil,
+		},
+		{
+			process: func([]byte) ([]Savable, error) {
+				return []Savable{&mockSavable{
+					ns: "hello",
+					k:  "world",
+					v:  []byte("!"),
+				}}, nil
+			},
+			save: func(string, string, []byte) error {
+				return errors.New("this one")
+			},
+			err: errors.New("this one"),
+		},
+		{
+			process: func([]byte) ([]Savable, error) {
+				return []Savable{}, errors.New("that one")
+			},
+			save: func(string, string, []byte) error {
+				return nil
+			},
+			err: errors.New("that one"),
+		},
+		{
+			process: func([]byte) ([]Savable, error) {
+				return []Savable{}, nil
+			},
+			save: func(string, string, []byte) error {
+				return nil
+			},
+			err: nil,
+		},
+	}
+	for _, c := range cases {
+		f, err := New(c.process, c.save)
+		if err != nil && err != c.err {
+			t.Errorf("cannot create new fetcher: %v", err)
+		} else if err == nil {
+			if err := f.FetchProcessSave(s.URL); err != nil && err.Error() != c.err.Error() {
+				t.Errorf("unexpected error: %v, expected %v", err, c.err)
+			}
+		}
+	}
+}
+
+func mockRemote() *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	}))
+}