email-xactions-to-todo/scrape.go

package main

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net/mail"
	"regexp"
	"strconv"
	"strings"
)

type scraper interface {
	scrape(*mail.Message) ([]*Transaction, error)
}

type chaseScraper struct{}
type citiScraper struct{}
type uccuScraper struct{}

func Scrape(m *mail.Message, banks map[Bank]bool) ([]*Transaction, error) {
	scraper, err := buildScraper(m, banks)
	if err != nil {
		return nil, err
	}
	return scraper.scrape(m)
}

func buildScraper(m *mail.Message, banks map[Bank]bool) (scraper, error) {
	subject := fmt.Sprint(m.Header["Subject"])
	if !containsAny(subject, "transaction", "report", "Transaction", "payment") {
		return nil, errors.New("cannot build scraper for subject " + subject)
	}
	from := fmt.Sprint(m.Header["From"])
	if strings.Contains(from, "Chase") && banks[Chase] {
		return newChaseScraper(), nil
	}
	if strings.Contains(from, "Citi") && banks[Citi] {
		return newCitiScraper(), nil
	}
	if strings.Contains(from, "Notifications@uccu.com") && banks[UCCU] {
		return newUCCUScraper(), nil
	}
	return nil, errors.New("unknown sender: " + from)
}

func newChaseScraper() scraper {
	return &chaseScraper{}
}

func newUCCUScraper() scraper {
	return &uccuScraper{}
}

func newCitiScraper() scraper {
	return &citiScraper{}
}

func containsAny(a string, b ...string) bool {
	for i := range b {
		if strings.Contains(a, b[i]) {
			return true
		}
	}
	return false
}

func (c *chaseScraper) scrape(m *mail.Message) ([]*Transaction, error) {
	transactions, err := c.scrape2021(m)
	if err == nil && len(transactions) > 0 {
		return transactions, err
	}
	return c.scrape2020(m)
}

func (c *chaseScraper) scrape2021(m *mail.Message) ([]*Transaction, error) {
	if t, err := c.scrape2021Payment(m); err == nil {
		return t, err
	}
	return c.scrape2021Charge(m)
}

func (c *chaseScraper) scrape2021Payment(m *mail.Message) ([]*Transaction, error) {
	re := regexp.MustCompile(`^We've received your .* payment$`)
	if !re.Match([]byte(m.Header["Subject"][0])) {
		return nil, errors.New("no match subject search")
	}

	b, err := ioutil.ReadAll(m.Body)
	if err != nil {
		return nil, err
	}

	re = regexp.MustCompile(`\$[0-9]+\.[0-9]{2}`)
	amount := "-" + strings.TrimLeft(string(re.Find(b)), "$")
	amount = strings.TrimLeft(string(re.Find(b)), "$")

	vendor := "Payment"

	re = regexp.MustCompile(`\(\.\.\.[0-9]{4}\)`)
	match := re.Find(b)
	re = regexp.MustCompile(`[0-9]{4}`)
	account := string(re.Find(match))

	return []*Transaction{NewTransaction(account, amount, vendor, fmt.Sprint(m.Header["Date"]), Chase)}, nil
}

func (c *chaseScraper) scrape2021Charge(m *mail.Message) ([]*Transaction, error) {
	re := regexp.MustCompile(`^Your \$(?P<amount>[0-9\.]*) transaction with (?P<vendor>.*)$`)
	matches := re.FindSubmatch([]byte(m.Header["Subject"][0]))
	if len(matches) < 1 {
		return nil, errors.New("no match subject search")
	}
	amount := string(matches[1])
	vendor := string(matches[2])

	b, _ := ioutil.ReadAll(m.Body)
	re = regexp.MustCompile(`\(\.\.\.[0-9]{4}\)`)
	match := re.Find(b)
	re = regexp.MustCompile(`[0-9]{4}`)
	account := string(re.Find(match))

	return []*Transaction{NewTransaction(account, amount, vendor, fmt.Sprint(m.Header["Date"]), Chase)}, nil
}

func (c *chaseScraper) scrape2020(m *mail.Message) ([]*Transaction, error) {
	b, err := ioutil.ReadAll(m.Body)
	if err != nil {
		return nil, err
	}
	re := regexp.MustCompile(`A charge of \([^)]*\) (?P<amount>[\d\.]+) at (?P<vendor>.*) has been authorized`)
	matches := re.FindSubmatch(b)
	if len(matches) < 2 {
		return nil, fmt.Errorf("no full matches found")
	}
	results := make(map[string][]string)
	for i, name := range re.SubexpNames() {
		if i != 0 && name != "" {
			results[name] = append(results[name], string(matches[i]))
		}
	}
	if len(results) != 2 || len(results["amount"]) != len(results["vendor"]) {
		return nil, fmt.Errorf("unexpected matches found looking for transactions: %+v", results)
	}
	re = regexp.MustCompile(`account ending in (?P<account>[0-9]{4})\.`)
	match := re.Find(b)
	re = regexp.MustCompile(`[0-9]{4}`)
	account := string(re.Find(match))
	transactions := make([]*Transaction, len(results["amount"]))
	for i := range results["amount"] {
		transactions[i] = NewTransaction(account, results["amount"][i], results["vendor"][i], fmt.Sprint(m.Header["Date"]), Chase)
	}
	return transactions, nil
}

func (c *citiScraper) scrape(m *mail.Message) ([]*Transaction, error) {
	date := fmt.Sprint(m.Header["Date"])
	b, err := ioutil.ReadAll(m.Body)
	if err != nil {
		return nil, err
	}
	re := regexp.MustCompile(`Citi Alert: A \$[0-9][0-9]*\.[0-9][0-9] transaction was made at .* on card ending in`)
	match := re.Find(b)
	if len(match) == 0 {
		return nil, nil
	}

	rePrice := regexp.MustCompile(`[0-9][0-9]*\.[0-9][0-9]`)
	price := rePrice.Find(match)

	vendor := bytes.Split(bytes.Split(match, []byte(" on card ending in"))[0], []byte("transaction was made at "))[1]

	transaction := NewTransaction(Citi.String(), string(price), string(vendor), date, Citi)

	return []*Transaction{transaction}, nil
	//Citi Alert: A $598.14 transaction was made at REMIX MUSIC SPRINGDA on card ending in 3837
	/*
		b, err := ioutil.ReadAll(m.Body)
		if err != nil {
			return nil, err
		}
		targetLineRegexp := regexp.MustCompile(`Account #: XXXX[0-9]{4} .*`)
		targetMatches := targetLineRegexp.FindAll(b, -1)
		if len(targetMatches) == 0 {
			return nil, errors.New("no lines with transactions found")
		}

		results := make(map[string][]string)
		for _, b := range targetMatches {
			// Account #: XXXX3837 $137.87 at AMZN Mktp US           Amzn.com/bill WA on 04/03/2020, 09:05 PM ET
			regexp := regexp.MustCompile(`Account #: XXXX[0-9]{4} \$(?P<amount>[0-9]+\.[0-9]*) at (?P<account>[^,]*)`)
			matches := regexp.FindSubmatch(b)
			if len(matches) < 2 {
				return nil, fmt.Errorf("no full matches found: %s", b)
			}
			for i, name := range regexp.SubexpNames() {
				if i != 0 && name != "" {
					if name == "account" {
						matches[i] = bytes.Split(matches[i], []byte(" on "))[0]
					}
					results[name] = append(results[name], string(matches[i]))
				}
			}
			if len(results) != 2 || len(results["amount"]) != len(results["account"]) {
				return nil, fmt.Errorf("unexpected matches found looking for transactions: %+v", results)
			}
		}

		transactions := make([]*Transaction, len(results["amount"]))
		for i := range results["amount"] {
			transactions[i] = NewTransaction(Citi.String(), results["amount"][i], results["account"][i], fmt.Sprint(m.Header["Date"]), Citi)
		}

		return transactions, nil
	*/
}

func (c *uccuScraper) scrape(m *mail.Message) ([]*Transaction, error) {
	b, err := ioutil.ReadAll(m.Body)
	if err != nil {
		return nil, err
	}
	regexp := regexp.MustCompile(`\$([0-9]+,?)+\.[0-9][0-9]`)
	match := regexp.Find(b)
	if len(match) == 0 {
		return nil, fmt.Errorf("no matches found")
	}
	match = match[1:]
	match = bytes.ReplaceAll(match, []byte(","), []byte{})
	f, err := strconv.ParseFloat(string(match), 10)
	if err != nil {
		return nil, err
	}
	if !bytes.Contains(b, []byte("credit")) {
		f *= -1.0
	}
	transaction := NewTransaction(UCCU.String(), fmt.Sprintf("%.2f", f), "?", fmt.Sprint(m.Header["Date"]), UCCU)
	return []*Transaction{transaction}, nil
}