diff --git a/fetch/fetch.go b/fetch/fetch.go new file mode 100644 index 0000000..0272d6b --- /dev/null +++ b/fetch/fetch.go @@ -0,0 +1,53 @@ +package fetch + +import ( + "io/ioutil" + "net/http" +) + +type Savable interface { + Namespace() string + Key() string + Value() []byte +} + +type Fetch struct { + process func([]byte) ([]Savable, error) + save func(string, string, []byte) error + client *http.Client +} + +func New(process func([]byte) ([]Savable, error), save func(string, string, []byte) error) (*Fetch, error) { + return &Fetch{ + save: save, + process: process, + client: &http.Client{}, + }, nil +} + +func (fetcher *Fetch) FetchProcessSave(url string) error { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + resp, err := fetcher.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + items, err := fetcher.process(b) + if err != nil { + return err + } + for i := range items { + err := fetcher.save(items[i].Namespace(), items[i].Key(), items[i].Value()) + if err != nil { + return err + } + } + return nil +} diff --git a/fetch/fetch_test.go b/fetch/fetch_test.go new file mode 100644 index 0000000..e672d1d --- /dev/null +++ b/fetch/fetch_test.go @@ -0,0 +1,88 @@ +package fetch + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +type mockSavable struct { + ns string + k string + v []byte +} + +func (ms *mockSavable) Namespace() string { return ms.ns } +func (ms *mockSavable) Key() string { return ms.k } +func (ms *mockSavable) Value() []byte { return ms.v } + +func Test_Fetch(t *testing.T) { + s := mockRemote() + defer s.Close() + cases := []struct { + process func([]byte) ([]Savable, error) + save func(string, string, []byte) error + err error + }{ + { + process: func([]byte) ([]Savable, error) { + return []Savable{&mockSavable{ + ns: "hello", + k: "world", + v: []byte("!"), + }}, nil + }, + save: func(string, string, []byte) error { + return nil + }, + err: nil, + }, + { + process: func([]byte) ([]Savable, error) { + return []Savable{&mockSavable{ + ns: "hello", + k: "world", + v: []byte("!"), + }}, nil + }, + save: func(string, string, []byte) error { + return errors.New("this one") + }, + err: errors.New("this one"), + }, + { + process: func([]byte) ([]Savable, error) { + return []Savable{}, errors.New("that one") + }, + save: func(string, string, []byte) error { + return nil + }, + err: errors.New("that one"), + }, + { + process: func([]byte) ([]Savable, error) { + return []Savable{}, nil + }, + save: func(string, string, []byte) error { + return nil + }, + err: nil, + }, + } + for _, c := range cases { + f, err := New(c.process, c.save) + if err != nil && err != c.err { + t.Errorf("cannot create new fetcher: %v", err) + } else if err == nil { + if err := f.FetchProcessSave(s.URL); err != nil && err.Error() != c.err.Error() { + t.Errorf("unexpected error: %v, expected %v", err, c.err) + } + } + } +} + +func mockRemote() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + })) +}