Implement citi scrape

config.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
+	"io/ioutil"
 	"local/args"
 	"local/oauth2"
 	"local/storage"
@@ -46,48 +48,12 @@ func NewConfig() Config {
 
 	token := as.GetString("todotoken")
 	if len(token) == 0 {
-		c := &http.Client{CheckRedirect: func(r *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
-		}}
-		body := "username=" + as.GetString("todopass")
-		req, err := http.NewRequest("POST", as.GetString("authaddr")+"/authorize/todo-server?"+oauth2.REDIRECT+"=127.0.0.1", strings.NewReader(body))
-		if err != nil {
-			panic(err)
-		}
-		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-		resp, err := c.Do(req)
-		if err != nil {
-			panic(err)
-		}
-		defer resp.Body.Close()
-		cookie := resp.Header.Get("Set-Cookie")
-		token = cookie[strings.Index(cookie, "=")+1:]
-		token = strings.Split(token, "; ")[0]
+		token = getToken(as)
 	}
+
 	list := as.GetString("todolist")
 	if len(list) == 0 {
-		req, err := http.NewRequest("GET", as.GetString("todoaddr")+"/ajax.php?loadLists", nil)
-		if err != nil {
-			panic(err)
-		}
-		req.Header.Set("Cookie", oauth2.COOKIE+"="+token)
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			panic(err)
-		}
-		defer resp.Body.Close()
-		var r struct {
-			List []struct {
-				ID string `json:"id"`
-			} `json:"list"`
-		}
-		if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
-			panic(err)
-		}
-		if len(r.List) == 0 {
-			panic("no lists found")
-		}
-		list = r.List[0].ID
+		list = getList(as, token)
 	}
 
 	storage, err := storage.New(storage.TypeFromString(as.GetString("store")), as.GetString("storeaddr"), as.GetString("storeuser"), as.GetString("storepass"))
@@ -107,3 +73,66 @@ func NewConfig() Config {
 	}
 	return config
 }
+
+func getToken(as *args.ArgSet) string {
+	c := &http.Client{CheckRedirect: func(r *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
+	}}
+	body := "username=" + as.GetString("todopass")
+	name := strings.Split(as.GetString("todoaddr"), ".")[0]
+	name = strings.TrimPrefix(name, "http://")
+	name = strings.TrimPrefix(name, "https://")
+	req, err := http.NewRequest("POST", as.GetString("authaddr")+"/authorize/"+name+"?"+oauth2.REDIRECT+"=127.0.0.1", strings.NewReader(body))
+	if err != nil {
+		panic(err)
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	resp, err := c.Do(req)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode > 399 {
+		panic("bad status getting token: " + resp.Status)
+	}
+	cookie := resp.Header.Get("Set-Cookie")
+	token := cookie[strings.Index(cookie, "=")+1:]
+	token = strings.Split(token, "; ")[0]
+	if len(token) == 0 {
+		panic(fmt.Sprintf("no token found: (%v) %v", resp.StatusCode, resp.Header))
+	}
+	return token
+}
+
+func getList(as *args.ArgSet, token string) string {
+	req, err := http.NewRequest("GET", as.GetString("todoaddr")+"/ajax.php?loadLists", nil)
+	if err != nil {
+		panic(err)
+	}
+	req.Header.Set("Cookie", oauth2.COOKIE+"="+token)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+	var r struct {
+		List []struct {
+			ID string `json:"id"`
+		} `json:"list"`
+	}
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		panic(err)
+	}
+	if err := json.Unmarshal(b, &r); err != nil {
+		panic(fmt.Errorf("%v: %s", err, b))
+	}
+	if len(r.List) == 0 {
+		panic("no lists found")
+	}
+	list := r.List[0].ID
+	if len(list) == 0 {
+		panic("empty list found")
+	}
+	return list
+}

scrape.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -64,7 +65,7 @@ func (c *chaseScraper) scrape(m *mail.Message) ([]*Transaction, error) {
 	regexp := regexp.MustCompile(`A charge of \([^)]*\) (?P<amount>[\d\.]+) at (?P<account>.*) has been authorized`)
 	matches := regexp.FindSubmatch(b)
 	if len(matches) < 2 {
-		return nil, fmt.Errorf("no matches found: %+v: %s", matches, b)
+		return nil, fmt.Errorf("no full matches found")
 	}
 	results := make(map[string][]string)
 	for i, name := range regexp.SubexpNames() {
@@ -83,5 +84,41 @@ func (c *chaseScraper) scrape(m *mail.Message) ([]*Transaction, error) {
 }
 
 func (c *citiScraper) scrape(m *mail.Message) ([]*Transaction, error) {
-	panic("not impl")
+	b, err := ioutil.ReadAll(m.Body)
+	if err != nil {
+		return nil, err
+	}
+	targetLineRegexp := regexp.MustCompile(`Account #: XXXX[0-9]{4} .*`)
+	targetMatches := targetLineRegexp.FindAll(b, -1)
+	if len(targetMatches) == 0 {
+		return nil, errors.New("no lines with transactions found")
+	}
+
+	results := make(map[string][]string)
+	for _, b := range targetMatches {
+		// Account #: XXXX3837 $137.87 at AMZN Mktp US           Amzn.com/bill WA on 04/03/2020, 09:05 PM ET
+		regexp := regexp.MustCompile(`Account #: XXXX[0-9]{4} \$(?P<amount>[0-9]+\.[0-9]*) at (?P<account>[^,]*)`)
+		matches := regexp.FindSubmatch(b)
+		if len(matches) < 2 {
+			return nil, fmt.Errorf("no full matches found: %s", b)
+		}
+		for i, name := range regexp.SubexpNames() {
+			if i != 0 && name != "" {
+				if name == "account" {
+					matches[i] = bytes.Split(matches[i], []byte(" on "))[0]
+				}
+				results[name] = append(results[name], string(matches[i]))
+			}
+		}
+		if len(results) != 2 || len(results["amount"]) != len(results["account"]) {
+			return nil, fmt.Errorf("unexpected matches found looking for transactions: %+v", results)
+		}
+	}
+
+	transactions := make([]*Transaction, len(results["amount"]))
+	for i := range results["amount"] {
+		transactions[i] = NewTransaction(results["amount"][i], results["account"][i], fmt.Sprint(m.Header["Date"]), Citi)
+	}
+
+	return transactions, nil
 }

transaction.go

@@ -3,6 +3,9 @@ package main
 import (
 	"crypto/md5"
 	"fmt"
+	"log"
+	"regexp"
+	"time"
 )
 
 type Transaction struct {
@@ -13,17 +16,37 @@ type Transaction struct {
 	Date    string
 }
 
+func (t *Transaction) Format() string {
+	return fmt.Sprintf("(%s) %v: %s @ %s", cleanDate(t.Date), t.Bank, t.Amount, t.Account)
+}
+
 func (t *Transaction) String() string {
 	return fmt.Sprint(*t)
 }
 
 func NewTransaction(amount, account, date string, bank Bank) *Transaction {
+	regexp := regexp.MustCompile(`\s\s+`)
 	t := &Transaction{
-		Amount:  amount,
-		Account: account,
+		Amount:  regexp.ReplaceAllString(amount, " "),
+		Account: regexp.ReplaceAllString(account, " "),
 		Bank:    bank,
 		Date:    date,
 	}
 	t.ID = fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprint(t))))
 	return t
 }
+
+func cleanDate(date string) string {
+	regexp := regexp.MustCompile(`[A-Z][a-z]{2}, [0-9][0-9]? [A-Z][a-z]{2} 2[0-9]{3}`)
+	matches := regexp.FindAllString(date, -1)
+	if len(matches) < 1 {
+		return date
+	}
+	date = matches[0]
+	time, err := time.Parse(`Mon, 2 Jan 2006`, date)
+	if err != nil {
+		log.Println(err)
+		return date
+	}
+	return time.Format("Mon Jan 2")
+}

transaction_test.go

@@ -0,0 +1,9 @@
+package main
+
+import "testing"
+
+func TestTransactionFormat(t *testing.T) {
+	x := NewTransaction("12.34", "Amazon", "[Wed, 1 Apr 2020 10:14:11 -0400 (EDT)]", Chase)
+	t.Logf("%s", x.String())
+	t.Logf("%s", x.Format())
+}

upload.go

@@ -12,7 +12,7 @@ import (
 func Upload(config Config, transaction *Transaction) error {
 	params := url.Values{
 		"list":  {config.TodoList},
-		"title": {fmt.Sprintf("%v: %s @ %s @ %s", transaction.Bank, transaction.Amount, transaction.Account, transaction.Date)},
+		"title": {transaction.Format()},
 		"tag":   {config.TodoTag},
 	}
 	req, err := http.NewRequest("POST", config.TodoAddr+"/ajax.php?newTask", strings.NewReader(params.Encode()))