54 lines
985 B
Go
54 lines
985 B
Go
package fetch
|
|
|
|
import (
|
|
"io/ioutil"
|
|
"net/http"
|
|
)
|
|
|
|
type Savable interface {
|
|
Namespace() string
|
|
Key() string
|
|
Value() []byte
|
|
}
|
|
|
|
type Fetch struct {
|
|
process func([]byte) ([]Savable, error)
|
|
save func(string, string, []byte) error
|
|
client *http.Client
|
|
}
|
|
|
|
func New(process func([]byte) ([]Savable, error), save func(string, string, []byte) error) (*Fetch, error) {
|
|
return &Fetch{
|
|
save: save,
|
|
process: process,
|
|
client: &http.Client{},
|
|
}, nil
|
|
}
|
|
|
|
func (fetcher *Fetch) FetchProcessSave(url string) error {
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp, err := fetcher.client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
b, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
items, err := fetcher.process(b)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for i := range items {
|
|
err := fetcher.save(items[i].Namespace(), items[i].Key(), items[i].Value())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|