gogs
/
email-xactions-to-todo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement citi scrape

master v0.3
bel 2020-04-04 20:44:30 +00:00
parent 9f3f3dc08f
commit f5a40f7890
3 changed files with 110 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
config.go
View File

@ -2,6 +2,8 @@ package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"local/args"
"local/oauth2"
"local/storage"
@ -46,48 +48,12 @@ func NewConfig() Config {
token := as.GetString("todotoken")
if len(token) == 0 {
c := &http.Client{CheckRedirect: func(r *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}}
body := "username=" + as.GetString("todopass")
req, err := http.NewRequest("POST", as.GetString("authaddr")+"/authorize/todo-server?"+oauth2.REDIRECT+"=127.0.0.1", strings.NewReader(body))
if err != nil {
panic(err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := c.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
cookie := resp.Header.Get("Set-Cookie")
token = cookie[strings.Index(cookie, "=")+1:]
token = strings.Split(token, "; ")[0]
token = getToken(as)
}
list := as.GetString("todolist")
if len(list) == 0 {
req, err := http.NewRequest("GET", as.GetString("todoaddr")+"/ajax.php?loadLists", nil)
if err != nil {
panic(err)
}
req.Header.Set("Cookie", oauth2.COOKIE+"="+token)
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
var r struct {
List []struct {
ID string `json:"id"`
} `json:"list"`
}
if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
panic(err)
}
if len(r.List) == 0 {
panic("no lists found")
}
list = r.List[0].ID
list = getList(as, token)
}
storage, err := storage.New(storage.TypeFromString(as.GetString("store")), as.GetString("storeaddr"), as.GetString("storeuser"), as.GetString("storepass"))
@ -107,3 +73,66 @@ func NewConfig() Config {
}
return config
}
func getToken(as *args.ArgSet) string {
c := &http.Client{CheckRedirect: func(r *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}}
body := "username=" + as.GetString("todopass")
name := strings.Split(as.GetString("todoaddr"), ".")[0]
name = strings.TrimPrefix(name, "http://")
name = strings.TrimPrefix(name, "https://")
req, err := http.NewRequest("POST", as.GetString("authaddr")+"/authorize/"+name+"?"+oauth2.REDIRECT+"=127.0.0.1", strings.NewReader(body))
if err != nil {
panic(err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := c.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
if resp.StatusCode > 399 {
panic("bad status getting token: " + resp.Status)
}
cookie := resp.Header.Get("Set-Cookie")
token := cookie[strings.Index(cookie, "=")+1:]
token = strings.Split(token, "; ")[0]
if len(token) == 0 {
panic(fmt.Sprintf("no token found: (%v) %v", resp.StatusCode, resp.Header))
}
return token
}
func getList(as *args.ArgSet, token string) string {
req, err := http.NewRequest("GET", as.GetString("todoaddr")+"/ajax.php?loadLists", nil)
if err != nil {
panic(err)
}
req.Header.Set("Cookie", oauth2.COOKIE+"="+token)
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
var r struct {
List []struct {
ID string `json:"id"`
} `json:"list"`
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
if err := json.Unmarshal(b, &r); err != nil {
panic(fmt.Errorf("%v: %s", err, b))
}
if len(r.List) == 0 {
panic("no lists found")
}
list := r.List[0].ID
if len(list) == 0 {
panic("empty list found")
}
return list
}

41
scrape.go
View File

@ -1,6 +1,7 @@
package main
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
@ -64,7 +65,7 @@ func (c *chaseScraper) scrape(m *mail.Message) ([]*Transaction, error) {
regexp := regexp.MustCompile(`A charge of \([^)]*\) (?P<amount>[\d\.]+) at (?P<account>.*) has been authorized`)
matches := regexp.FindSubmatch(b)
if len(matches) < 2 {
return nil, fmt.Errorf("no matches found: %+v: %s", matches, b)
return nil, fmt.Errorf("no full matches found")
}
results := make(map[string][]string)
for i, name := range regexp.SubexpNames() {
@ -83,5 +84,41 @@ func (c *chaseScraper) scrape(m *mail.Message) ([]*Transaction, error) {
}
func (c *citiScraper) scrape(m *mail.Message) ([]*Transaction, error) {
panic("not impl")
b, err := ioutil.ReadAll(m.Body)
if err != nil {
return nil, err
}
targetLineRegexp := regexp.MustCompile(`Account #: XXXX[0-9]{4} .*`)
targetMatches := targetLineRegexp.FindAll(b, -1)
if len(targetMatches) == 0 {
return nil, errors.New("no lines with transactions found")
}
results := make(map[string][]string)
for _, b := range targetMatches {
// Account #: XXXX3837 $137.87 at AMZN Mktp US Amzn.com/bill WA on 04/03/2020, 09:05 PM ET
regexp := regexp.MustCompile(`Account #: XXXX[0-9]{4} \$(?P<amount>[0-9]+\.[0-9]*) at (?P<account>[^,]*)`)
matches := regexp.FindSubmatch(b)
if len(matches) < 2 {
return nil, fmt.Errorf("no full matches found: %s", b)
}
for i, name := range regexp.SubexpNames() {
if i != 0 && name != "" {
if name == "account" {
matches[i] = bytes.Split(matches[i], []byte(" on "))[0]
}
results[name] = append(results[name], string(matches[i]))
}
}
if len(results) != 2 || len(results["amount"]) != len(results["account"]) {
return nil, fmt.Errorf("unexpected matches found looking for transactions: %+v", results)
}
}
transactions := make([]*Transaction, len(results["amount"]))
for i := range results["amount"] {
transactions[i] = NewTransaction(results["amount"][i], results["account"][i], fmt.Sprint(m.Header["Date"]), Citi)
}
return transactions, nil
}

5
transaction.go
View File

@ -25,9 +25,10 @@ func (t *Transaction) String() string {
}
func NewTransaction(amount, account, date string, bank Bank) *Transaction {
regexp := regexp.MustCompile(`\s\s+`)
t := &Transaction{
Amount: amount,
Account: account,
Amount: regexp.ReplaceAllString(amount, " "),
Account: regexp.ReplaceAllString(account, " "),
Bank: bank,
Date: date,
}