package main import ( "bytes" "errors" "fmt" "io" "io/ioutil" "net/mail" "regexp" "slices" "strconv" "strings" ) type scraper interface { scrape(*mail.Message) ([]*Transaction, error) } type fidelityScraper struct{} type bankOfAmericaScraper struct{} type chaseScraper struct{} type citiScraper struct{} type uccuScraper struct{} type amexScraper struct{} type plumasScraper struct{} func Scrape(m *mail.Message, banks map[Bank]bool) ([]*Transaction, error) { scraper, err := buildScraper(m, banks) if err != nil { return nil, err } return scraper.scrape(m) } func buildScraper(m *mail.Message, banks map[Bank]bool) (scraper, error) { subject := fmt.Sprint(m.Header["Subject"]) if !containsAny(subject, "transaction", "report", "Transaction", "payment", "Payment", "Deposit", "Withdrawal", "balance is", "Balance", "Large Purchase", "transaction with", "credit pending on your card", "Account Alert") { return nil, errors.New("cannot build scraper for subject " + subject) } from := fmt.Sprint(m.Header["From"]) if strings.Contains(from, "Chase") && banks[Chase] { return newChaseScraper(), nil } if strings.Contains(from, "Fidelity") && banks[Fidelity] { return newFidelityScraper(), nil } if strings.Contains(from, "Bank of America") && banks[BankOfAmerica] { return newBankOfAmericaScraper(), nil } if strings.Contains(from, "Citi") && banks[Citi] { return newCitiScraper(), nil } if strings.Contains(from, "Notifications@uccu.com") && banks[UCCU] { return newUCCUScraper(), nil } if strings.Contains(from, "Notifications@plumasbank.com") && banks[Plumas] { return newPlumasScraper(), nil } if strings.Contains(strings.ToLower(from), strings.ToLower("AmericanExpress")) && banks[Amex] { return newAmexScraper(), nil } return nil, fmt.Errorf("unknown sender: %q", from) } func newFidelityScraper() scraper { return &fidelityScraper{} } func newBankOfAmericaScraper() scraper { return &bankOfAmericaScraper{} } func newChaseScraper() scraper { return &chaseScraper{} } func newUCCUScraper() scraper { return &uccuScraper{} } func newCitiScraper() scraper { return &citiScraper{} } func newAmexScraper() scraper { return &amexScraper{} } func newPlumasScraper() scraper { return &plumasScraper{} } func containsAny(a string, b ...string) bool { for i := range b { if strings.Contains(a, b[i]) { return true } } return false } func (c *chaseScraper) scrape(m *mail.Message) ([]*Transaction, error) { if transactions, err := c.scrape202506(m); err == nil && len(transactions) > 0 { return transactions, err } if transactions, err := c.scrape2025Balance(m); err == nil && len(transactions) > 0 { return transactions, err } if transactions, err := c.scrape2021(m); err == nil && len(transactions) > 0 { return transactions, err } return c.scrape2020(m) } func (c *chaseScraper) scrape202506(m *mail.Message) ([]*Transaction, error) { if xactions, err := c.scrape202506Charge(m); err == nil && len(xactions) > 0 { return xactions, nil } return c.scrape202506Credit(m) } func (c *chaseScraper) scrape202506Credit(m *mail.Message) ([]*Transaction, error) { pattern := `^You have a \$([0-9][,0-9]*\.[0-9]{2}) credit pending on your credit card` re := regexp.MustCompile(pattern) matches := re.FindSubmatch([]byte(m.Header["Subject"][0])) if len(matches) < 1 { return nil, fmt.Errorf("no match subject search (%q vs %q)", pattern, m.Header["Subject"][0]) } b, _ := io.ReadAll(m.Body) amount := string(matches[1]) amount = strings.ReplaceAll(amount, ",", "") vendor := "*" re = regexp.MustCompile(`\(\.\.\.[0-9]{4}\)`) match := re.Find(b) re = regexp.MustCompile(`[0-9]{4}`) account := string(re.Find(match)) return []*Transaction{NewTransaction(account, amount, vendor, fmt.Sprint(m.Header["Date"]), Chase)}, nil } func (c *chaseScraper) scrape202506Charge(m *mail.Message) ([]*Transaction, error) { pattern := `^(Your|You made an?) (\$(?P[,0-9\.]*)|online, phone, or mail) transaction with (?P.*)$` re := regexp.MustCompile(pattern) matches := re.FindSubmatch([]byte(m.Header["Subject"][0])) if len(matches) < 1 { return nil, fmt.Errorf("no match subject search (%q vs %q)", pattern, m.Header["Subject"][0]) } b, _ := io.ReadAll(m.Body) amount := string(matches[3]) if amount == "" { pattern := `\$([0-9][,0-9]*\.[0-9]{2})` matches := regexp.MustCompile(pattern).FindStringSubmatch(string(b)) if len(matches) < 2 || len(matches[1]) == 0 { return nil, fmt.Errorf("no $x,yyy,zz in body") } amount = matches[1] } amount = strings.ReplaceAll(amount, ",", "") vendor := string(matches[4]) re = regexp.MustCompile(`\(\.\.\.[0-9]{4}\)`) match := re.Find(b) re = regexp.MustCompile(`[0-9]{4}`) account := string(re.Find(match)) return []*Transaction{NewTransaction(account, amount, vendor, fmt.Sprint(m.Header["Date"]), Chase)}, nil } func (c *chaseScraper) scrape2025Balance(m *mail.Message) ([]*Transaction, error) { re := regexp.MustCompile(`^Your.*balance is \$[0-9,\.]*$`) if !re.Match([]byte(m.Header["Subject"][0])) { return nil, errors.New("no match subject search") } subject := m.Header["Subject"][0] fields := strings.Fields(subject) amount := fields[len(fields)-1] amount = strings.TrimLeft(amount, "$") amount = strings.ReplaceAll(amount, ",", "") amount = fmt.Sprintf("=%s", amount) b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } re = regexp.MustCompile(`\(\.\.\.([0-9]{4})\)`) submatches := re.FindSubmatch(b) account := string(submatches[len(submatches)-1]) return []*Transaction{NewTransaction(account, amount, "*", fmt.Sprint(m.Header["Date"]), Chase)}, nil } func (c *chaseScraper) scrape2021(m *mail.Message) ([]*Transaction, error) { if t, err := c.scrape2021Payment(m); err == nil { return t, err } return c.scrape2021Charge(m) } func (c *chaseScraper) scrape2021Payment(m *mail.Message) ([]*Transaction, error) { re := regexp.MustCompile(`^We've received your .* payment$`) if !re.Match([]byte(m.Header["Subject"][0])) { return nil, errors.New("no match subject search") } b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } re = regexp.MustCompile(`\$[0-9,]+\.[0-9]{2}`) amount := "-" + strings.TrimLeft(string(re.Find(b)), "$") amount = strings.TrimLeft(string(re.Find(b)), "$") amount = strings.ReplaceAll(amount, ",", "") vendor := "Payment" re = regexp.MustCompile(`\(\.\.\.[0-9]{4}\)`) match := re.Find(b) re = regexp.MustCompile(`[0-9]{4}`) account := string(re.Find(match)) return []*Transaction{NewTransaction(account, amount, vendor, fmt.Sprint(m.Header["Date"]), Chase)}, nil } func (c *chaseScraper) scrape2021Charge(m *mail.Message) ([]*Transaction, error) { re := regexp.MustCompile(`^Your \$(?P[,0-9\.]*) transaction with (?P.*)$`) matches := re.FindSubmatch([]byte(m.Header["Subject"][0])) if len(matches) < 1 { return nil, errors.New("no match subject search") } amount := string(matches[1]) amount = strings.ReplaceAll(amount, ",", "") vendor := string(matches[2]) b, _ := ioutil.ReadAll(m.Body) re = regexp.MustCompile(`\(\.\.\.[0-9]{4}\)`) match := re.Find(b) re = regexp.MustCompile(`[0-9]{4}`) account := string(re.Find(match)) return []*Transaction{NewTransaction(account, amount, vendor, fmt.Sprint(m.Header["Date"]), Chase)}, nil } func (c *chaseScraper) scrape2020(m *mail.Message) ([]*Transaction, error) { b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } re := regexp.MustCompile(`A charge of \([^)]*\) (?P[\d\.]+) at (?P.*) has been authorized`) matches := re.FindSubmatch(b) if len(matches) < 2 { return nil, fmt.Errorf("no full matches found") } results := make(map[string][]string) for i, name := range re.SubexpNames() { if i != 0 && name != "" { results[name] = append(results[name], string(matches[i])) } } if len(results) != 2 || len(results["amount"]) != len(results["vendor"]) { return nil, fmt.Errorf("unexpected matches found looking for transactions: %+v", results) } re = regexp.MustCompile(`account ending in (?P[0-9]{4})\.`) match := re.Find(b) re = regexp.MustCompile(`[0-9]{4}`) account := string(re.Find(match)) transactions := make([]*Transaction, len(results["amount"])) for i := range results["amount"] { transactions[i] = NewTransaction(account, results["amount"][i], results["vendor"][i], fmt.Sprint(m.Header["Date"]), Chase) } return transactions, nil } func (c *citiScraper) scrape(m *mail.Message) ([]*Transaction, error) { date := fmt.Sprint(m.Header["Date"]) b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } re := regexp.MustCompile(`Citi Alert: A \$[,0-9][,0-9]*\.[0-9][0-9] transaction was made at .* on card ending in`) match := re.Find(b) if len(match) == 0 { return nil, nil } rePrice := regexp.MustCompile(`[0-9][0-9,]*\.[0-9][0-9]`) price := rePrice.Find(match) price = []byte(strings.ReplaceAll(string(price), ",", "")) vendor := bytes.Split(bytes.Split(match, []byte(" on card ending in"))[0], []byte("transaction was made at "))[1] transaction := NewTransaction(Citi.String(), string(price), string(vendor), date, Citi) return []*Transaction{transaction}, nil } func (c *uccuScraper) scrape(m *mail.Message) ([]*Transaction, error) { b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } if bytes.Contains(b, []byte(`alance alert`)) { return c.scrapeBalance(m, b) } regexp := regexp.MustCompile(`\$([0-9]+,?)+\.[0-9][0-9]`) match := regexp.Find(b) if len(match) == 0 { return nil, fmt.Errorf("no matches found") } match = match[1:] match = bytes.ReplaceAll(match, []byte(","), []byte{}) f, err := strconv.ParseFloat(string(match), 10) if err != nil { return nil, err } if !bytes.Contains(b, []byte("credit")) { f *= -1.0 } transaction := NewTransaction(UCCU.String(), fmt.Sprintf("%.2f", f), "?", fmt.Sprint(m.Header["Date"]), UCCU) return []*Transaction{transaction}, nil } func (c *uccuScraper) scrapeBalance(m *mail.Message, b []byte) ([]*Transaction, error) { re := regexp.MustCompile(`is \$([0-9]+,?)+\.[0-9][0-9]`) match := re.Find(b) if len(match) == 0 { return nil, fmt.Errorf("no matches found") } match = match[4:] match = bytes.ReplaceAll(match, []byte(","), []byte{}) f, err := strconv.ParseFloat(string(match), 10) if err != nil { return nil, err } acc := UCCU.String() re = regexp.MustCompile(`ending in [0-9]*`) match = re.Find(b) if len(match) > 0 { acc = fmt.Sprintf("%s-%s", acc, match[len(`ending in `):]) } transaction := NewTransaction(acc, fmt.Sprintf("=%.2f", f), "*", fmt.Sprint(m.Header["Date"]), UCCU) return []*Transaction{transaction}, nil } func (c *amexScraper) scrape(m *mail.Message) ([]*Transaction, error) { b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } b = bytes.ReplaceAll(b, []byte("=\n"), []byte("")) matches := regexp.MustCompile(`\$([0-9]+,?)+\.[0-9][0-9]`).FindAll(b, -1) matches = slices.DeleteFunc(matches, func(match []byte) bool { return string(match) == "$1.00" }) if len(matches) == 0 { return nil, fmt.Errorf("no matches found") } match := matches[0] match = match[1:] match = bytes.ReplaceAll(match, []byte(","), []byte{}) f, err := strconv.ParseFloat(string(match), 10) if err != nil { return nil, err } f *= -1.0 vendors := regexp.MustCompile(`>[A-Z][A-Z ]*<`).FindAll(b, -1) vendors = slices.DeleteFunc(vendors, func(b []byte) bool { return string(b) == ">BREE A LAPOINTE<" }) vendor := "*" if len(vendors) > 0 { vendor = string(vendors[0]) } vendor = strings.TrimSpace(strings.Trim(strings.Trim(vendor, ">"), "<")) vendor = strings.ReplaceAll(vendor, "\n", "") accs := regexp.MustCompile(`Account Ending: [0-9]*([0-9]{4})[^0-9]`).FindSubmatch(b) acc := "?" if len(accs) > 1 { acc = string(accs[1]) } acc = strings.ReplaceAll(acc, "\n", "") transaction := NewTransaction( fmt.Sprintf("%s-%s", Amex.String(), acc), fmt.Sprintf("%.2f", f), vendor, fmt.Sprint(m.Header["Date"]), Amex, ) return []*Transaction{transaction}, nil } func (c *fidelityScraper) scrape(m *mail.Message) ([]*Transaction, error) { subject := fmt.Sprint(m.Header["Subject"]) if strings.Contains(subject, "Daily Balance") { return c.scrapeBalance(m) } if strings.Contains(subject, "Debit Withdrawal") { return c.scrapeWithdrawal(m) } if strings.Contains(subject, "Deposit Received") { return c.scrapeDeposit(m) } panic(nil) } func (c *fidelityScraper) scrapeBalance(m *mail.Message) ([]*Transaction, error) { b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } fidelAcc, _ := findSubstringBetween(b, "Account: XXXXX", "\n") fidelAmount, _ := findSubstringBetween(b, "Your Daily Balance is $", " for ") transaction := NewTransaction( fmt.Sprintf("%s-%s", Fidelity, fidelAcc), "="+strings.ReplaceAll(fidelAmount, ",", ""), "*", fmt.Sprint(m.Header["Date"]), Fidelity, ) return []*Transaction{transaction}, nil } func (c *fidelityScraper) scrapeDeposit(m *mail.Message) ([]*Transaction, error) { b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } fidelAcc, _ := findSubstringBetween(b, "Account: XXXXX", "\n") transaction := NewTransaction( fmt.Sprintf("%s-%s", Fidelity, fidelAcc), "?.??", "misc", fmt.Sprint(m.Header["Date"]), Fidelity, ) return []*Transaction{transaction}, nil } func (c *fidelityScraper) scrapeWithdrawal(m *mail.Message) ([]*Transaction, error) { b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } amount, amountOk := findSubstringBetween(b, "in the amount of $", " ") fidelAcc, fidelAccOk := findSubstringBetween(b, "For account ending in ", ":") acc, accOk := findSubstringBetween(b, "in the amount of $"+amount+" by ", ".") if amount == "" || acc == "" { return nil, fmt.Errorf("no amount/account found: fidelAcc=%v,fidelAccOk=%v, acc=%v,accOk=%v, amount=%v,amountOk=%v", fidelAcc, fidelAccOk, acc, accOk, amount, amountOk) } transaction := NewTransaction(fmt.Sprintf("%s-%s", Fidelity, fidelAcc), amount, acc, fmt.Sprint(m.Header["Date"]), Fidelity) return []*Transaction{transaction}, nil } func (c *bankOfAmericaScraper) scrape(m *mail.Message) ([]*Transaction, error) { subject := fmt.Sprint(m.Header["Subject"]) if strings.Contains(subject, "Credit card transaction") { return c.scrapeCharge(m) } if strings.Contains(subject, "Credit Card Payment") { return c.scrapePayment(m) } return nil, errors.New("not impl") } func (c *bankOfAmericaScraper) scrapeCharge(m *mail.Message) ([]*Transaction, error) { b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } if v, err := c.scrapeCharge2023(m, b); err == nil { return v, nil } return c.scrapeCharge2024(m, b) } func (c *bankOfAmericaScraper) scrapeCharge2023(m *mail.Message, b []byte) ([]*Transaction, error) { amount := findFloatAfter(b, "Amount: $") acc := string(findLineAfter(b, "Where: ")) if amount == "" || acc == "" { return nil, errors.New("no amount/account found") } transaction := NewTransaction(BankOfAmerica.String(), amount, acc, fmt.Sprint(m.Header["Date"]), BankOfAmerica) return []*Transaction{transaction}, nil } func (c *bankOfAmericaScraper) scrapeCharge2024(m *mail.Message, b []byte) ([]*Transaction, error) { amountPattern := `Amount:[^\$]*\$([0-9]*\.[0-9]*)` amountMatches := regexp.MustCompile(amountPattern).FindSubmatch(bytes.ReplaceAll(b, []byte("\n"), []byte(" "))) if len(amountMatches) < 2 { return nil, fmt.Errorf("email does not match amount %q: %+v", amountPattern, amountMatches) } amount := string(amountMatches[1]) b2 := bytes.Split(b, []byte("Where:"))[1] b3 := bytes.Split(b2, []byte(""))[0] b4 := bytes.ReplaceAll(b3, []byte("\n"), []byte(" ")) accPattern := `(.*)` accMatches := regexp.MustCompile(accPattern).FindSubmatch(b4) if len(accMatches) < 2 { return nil, fmt.Errorf("email does not match acc %q", accPattern) } acc := string(accMatches[1]) if amount == "" || acc == "" { return nil, errors.New("no amount/account found") } transaction := NewTransaction(BankOfAmerica.String(), amount, acc, fmt.Sprint(m.Header["Date"]), BankOfAmerica) return []*Transaction{transaction}, nil } func (c *bankOfAmericaScraper) scrapePayment(m *mail.Message) ([]*Transaction, error) { b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } amount := "-" + findFloatAfter(b, "Payment: $") acc := "Payment" if amount == "" || acc == "" { return nil, errors.New("no amount/account found") } transaction := NewTransaction(BankOfAmerica.String(), amount, acc, fmt.Sprint(m.Header["Date"]), BankOfAmerica) return []*Transaction{transaction}, nil } func (c *plumasScraper) scrape(m *mail.Message) ([]*Transaction, error) { b, err := ioutil.ReadAll(m.Body) if err != nil { return nil, err } if bytes.Contains(b, []byte(`alance alert`)) { return c.scrapeBalance(m, b) } regexp := regexp.MustCompile(`\$([0-9]+,?)+\.[0-9][0-9]`) match := regexp.Find(b) if len(match) == 0 { return nil, fmt.Errorf("no matches found") } match = match[1:] match = bytes.ReplaceAll(match, []byte(","), []byte{}) f, err := strconv.ParseFloat(string(match), 10) if err != nil { return nil, err } if !bytes.Contains(b, []byte("credit")) { f *= -1.0 } transaction := NewTransaction(Plumas.String(), fmt.Sprintf("%.2f", f), "?", fmt.Sprint(m.Header["Date"]), Plumas) return []*Transaction{transaction}, nil } func (c *plumasScraper) scrapeBalance(m *mail.Message, b []byte) ([]*Transaction, error) { re := regexp.MustCompile(`is \$([0-9]+,?)+\.[0-9][0-9]`) match := re.Find(b) if len(match) == 0 { return nil, fmt.Errorf("no matches found") } match = match[4:] match = bytes.ReplaceAll(match, []byte(","), []byte{}) f, err := strconv.ParseFloat(string(match), 10) if err != nil { return nil, err } acc := Plumas.String() re = regexp.MustCompile(`ending in [0-9]*`) match = re.Find(b) if len(match) > 0 { acc = fmt.Sprintf("%s-%s", acc, match[len(`ending in `):]) } transaction := NewTransaction(acc, fmt.Sprintf("=%.2f", f), "*", fmt.Sprint(m.Header["Date"]), Plumas) return []*Transaction{transaction}, nil } func findSubstringBetween(b []byte, prefix, suffix string) (string, bool) { byPre := bytes.Split(b, []byte(prefix)) if len(byPre) < 2 { return "", false } bySuff := bytes.Split(byPre[1], []byte(suffix)) if len(bySuff) < 2 { return "", false } return string(bySuff[0]), true } func findFloatAfter(b []byte, prefix string) string { amount := string(findLineAfter(b, prefix)) words := strings.Split(amount, " ") lastword := words[len(words)-1] escapedfloat := strings.TrimPrefix(lastword, "$") fixEscape := strings.ReplaceAll(escapedfloat, "=2E", ".") amount = fixEscape return amount } func findLineAfter(b []byte, prefix string) []byte { for _, line := range bytes.Split(b, []byte("\n")) { if bytes.HasPrefix(line, []byte(prefix)) { return bytes.TrimSpace(bytes.TrimPrefix(line, []byte(prefix))) } } return nil }