upgrade to .todo. inbox

bank.go

@@ -7,10 +7,13 @@ const (
 	Citi          Bank = iota + 1
 	UCCU          Bank = iota + 1
 	BankOfAmerica Bank = iota + 1
+	Fidelity      Bank = iota + 1
 )
 
 func (b Bank) String() string {
 	switch b {
+	case Fidelity:
+		return "Fidelity"
 	case BankOfAmerica:
 		return "BankOfAmerica"
 	case Chase:

config.go

@@ -1,15 +1,11 @@
 package main
 
 import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"local/args"
-	"local/oauth2"
-	"local/storage"
 	"log"
-	"net/http"
 	"strings"
+
+	"gitea.inhome.blapointe.com/local/args"
+	"gitea.inhome.blapointe.com/local/storage"
 )
 
 type Uploader int
@@ -30,6 +26,7 @@ type Config struct {
 	EmailUser           string
 	EmailPass           string
 	EmailIMAP           string
+	EmailLimit          int
 	TodoAddr            string
 	TodoToken           string
 	TodoList            string
@@ -49,6 +46,7 @@ func NewConfig() Config {
 	as.Append(args.STRING, "emailuser", "email username", "breellocaldev@gmail.com")
 	as.Append(args.STRING, "emailpass", "email password", "diblloewfncwssof")
 	as.Append(args.STRING, "emailimap", "email imap", "imap.gmail.com:993")
+	as.Append(args.INT, "emaillimit", "email limit", 0)
 
 	as.Append(args.STRING, "uploader", "todo, ledger, pttodo", "todo")
 
@@ -58,7 +56,7 @@ func NewConfig() Config {
 	as.Append(args.STRING, "todolist", "todo list", "")
 	as.Append(args.STRING, "todotag", "todo tag", "expense")
 
-	as.Append(args.STRING, "banks", "uccu,citi,chase,bankofamerica", "uccu,citi,chase,bankofamerica")
+	as.Append(args.STRING, "banks", "uccu,citi,chase,bankofamerica,fidelity", "uccu,citi,chase,bankofamerica,fidelity")
 	as.Append(args.STRING, "accounts", "regex to include filter accounts", ".*")
 	as.Append(args.STRING, "not-accounts", "regex to exclude filter accounts", "zzzzzz")
 
@@ -87,6 +85,7 @@ func NewConfig() Config {
 		EmailUser:           as.GetString("emailuser"),
 		EmailPass:           as.GetString("emailpass"),
 		EmailIMAP:           as.GetString("emailimap"),
+		EmailLimit:          as.GetInt("emaillimit"),
 		TodoAddr:            as.GetString("todoaddr"),
 		TodoTag:             as.GetString("todotag"),
 		AccountsPattern:     as.GetString("accounts"),
@@ -98,6 +97,7 @@ func NewConfig() Config {
 			Chase:         strings.Contains(strings.ToLower(as.GetString("banks")), strings.ToLower(Chase.String())),
 			Citi:          strings.Contains(strings.ToLower(as.GetString("banks")), strings.ToLower(Citi.String())),
 			UCCU:          strings.Contains(strings.ToLower(as.GetString("banks")), strings.ToLower(UCCU.String())),
+			Fidelity:      strings.Contains(strings.ToLower(as.GetString("banks")), strings.ToLower(Fidelity.String())),
 		},
 	}
 	log.Printf("config: %+v", config)
@@ -120,64 +120,9 @@ func NewConfig() Config {
 }
 
 func getToken(as *args.ArgSet) string {
-	c := &http.Client{CheckRedirect: func(r *http.Request, via []*http.Request) error {
+	panic("DEAD")
-		return http.ErrUseLastResponse
-	}}
-	body := "username=" + as.GetString("todopass")
-	name := strings.Split(as.GetString("todoaddr"), ".")[0]
-	name = strings.TrimPrefix(name, "http://")
-	name = strings.TrimPrefix(name, "https://")
-	req, err := http.NewRequest("POST", as.GetString("authaddr")+"/authorize/"+name+"?"+oauth2.REDIRECT+"=127.0.0.1", strings.NewReader(body))
-	if err != nil {
-		panic(err)
-	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	resp, err := c.Do(req)
-	if err != nil {
-		panic(err)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode > 399 {
-		panic("bad status getting token: " + resp.Status)
-	}
-	cookie := resp.Header.Get("Set-Cookie")
-	token := cookie[strings.Index(cookie, "=")+1:]
-	token = strings.Split(token, "; ")[0]
-	if len(token) == 0 {
-		panic(fmt.Sprintf("no token found: (%v) %v", resp.StatusCode, resp.Header))
-	}
-	return token
 }
 
 func getList(as *args.ArgSet, token string) string {
-	req, err := http.NewRequest("GET", as.GetString("todoaddr")+"/ajax.php?loadLists", nil)
+	panic("DEAD")
-	if err != nil {
-		panic(err)
-	}
-	req.Header.Set("Cookie", oauth2.COOKIE+"="+token)
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		panic(err)
-	}
-	defer resp.Body.Close()
-	var r struct {
-		List []struct {
-			ID string `json:"id"`
-		} `json:"list"`
-	}
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		panic(err)
-	}
-	if err := json.Unmarshal(b, &r); err != nil {
-		panic(fmt.Errorf("%v: %s", err, b))
-	}
-	if len(r.List) == 0 {
-		panic("no lists found")
-	}
-	list := r.List[0].ID
-	if len(list) == 0 {
-		panic("empty list found")
-	}
-	return list
 }

go.mod

@@ -1,25 +1,22 @@
-module local/email-xactions-to-todo
+module gitea.inhome.blapointe.com/local/email-xactions-to-todo
 
 go 1.17
 
 require (
+	gitea.inhome.blapointe.com/local-sandbox/contact v0.0.2-0.20231109150121-14036702ee2a
+	gitea.inhome.blapointe.com/local/args v0.0.0-20231109145953-eb2e1c1b8d56
+	gitea.inhome.blapointe.com/local/storage v0.0.0-20231109151605-736d446d407d
 	github.com/google/uuid v1.3.0
-	local/args v0.0.0-00010101000000-000000000000
-	local/oauth2 v0.0.0-00010101000000-000000000000
-	local/sandbox/contact/contact v0.0.0-00010101000000-000000000000
-	local/storage v0.0.0-00010101000000-000000000000
 )
 
 require (
 	cloud.google.com/go v0.33.1 // indirect
+	gitea.inhome.blapointe.com/local/logb v0.0.0-20231109150430-1221d87a6dbc // indirect
 	github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
 	github.com/aws/aws-sdk-go v1.15.81 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
-	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect
-	github.com/buraksezer/consistent v0.9.0 // indirect
 	github.com/bytbox/go-pop3 v0.0.0-20120201222208-3046caf0763e // indirect
-	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/emersion/go-imap v1.2.0 // indirect
 	github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
@@ -52,24 +49,11 @@ require (
 	golang.org/x/net v0.0.0-20190522155817-f3200d17e092 // indirect
 	golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
-	golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2 // indirect
+	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
 	google.golang.org/api v0.0.0-20181120235003-faade3cbb06a // indirect
 	google.golang.org/appengine v1.3.0 // indirect
 	gopkg.in/ini.v1 v1.42.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	local/logb v0.0.0-00010101000000-000000000000 // indirect
 )
-
-replace local/args => ../args
-
-replace local/oauth2 => ../oauth2
-
-replace local/sandbox/contact/contact => ../sandbox/contact/contact
-
-replace local/storage => ../storage
-
-replace local/router => ../router
-
-replace local/logb => ../logb

go.sum

@@ -1,12 +1,18 @@
 bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
 cloud.google.com/go v0.33.1 h1:fmJQWZ1w9PGkHR1YL/P7HloDvqlmKQ4Vpb7PC2e+aCk=
 cloud.google.com/go v0.33.1/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+gitea.inhome.blapointe.com/local-sandbox/contact v0.0.2-0.20231109150121-14036702ee2a h1:vDt7kgsUwsI6fq7ObPUuUQ2CiIts3RaEsvcuwruspEY=
+gitea.inhome.blapointe.com/local-sandbox/contact v0.0.2-0.20231109150121-14036702ee2a/go.mod h1:3LBm8MXwR5D5Z8gwp5p39KmabJR/F7cxBSZwVHWTfQQ=
+gitea.inhome.blapointe.com/local/args v0.0.0-20231109145953-eb2e1c1b8d56 h1:zTGGZ77KLFagqUvDSgTOnm0qF+iSLwQWiEtGjb2jjlY=
+gitea.inhome.blapointe.com/local/args v0.0.0-20231109145953-eb2e1c1b8d56/go.mod h1:SqCOE3bE3wvrztVIQGHuyxHKfDjRKU9EWhBdkmkiwyc=
+gitea.inhome.blapointe.com/local/logb v0.0.0-20231109150430-1221d87a6dbc h1:u3akQkq12V8xWXlcDgjZxIK6hqo6f1eHd9KOxAKMoKc=
+gitea.inhome.blapointe.com/local/logb v0.0.0-20231109150430-1221d87a6dbc/go.mod h1:KwilawX4UgD4HxSJAVFEzkuckrnHeQrd49KwUX6GpYU=
+gitea.inhome.blapointe.com/local/storage v0.0.0-20231109151605-736d446d407d h1:SQq4hWImnvtrRfpPgOW4go+sBjMluuhRL/43b8L0yB4=
+gitea.inhome.blapointe.com/local/storage v0.0.0-20231109151605-736d446d407d/go.mod h1:TRK5z/XTT6jws++Q21Y8DQot+5vZGTNeHf+RjuY8aQk=
 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
 github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 h1:6X8iB881g299aNEv6KXrcjL31iLOH7yA6NXoQX+MbDg=
 github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
 github.com/a8m/tree v0.0.0-20180321023834-3cf936ce15d6/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=
@@ -18,14 +24,8 @@ github.com/aws/aws-sdk-go v1.15.81/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3A
 github.com/billziss-gh/cgofuse v1.1.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0=
-github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
-github.com/buraksezer/consistent v0.9.0 h1:Zfs6bX62wbP3QlbPGKUhqDw7SmNkOzY5bHZIYXYpR5g=
-github.com/buraksezer/consistent v0.9.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw=
 github.com/bytbox/go-pop3 v0.0.0-20120201222208-3046caf0763e h1:mQTN05gz0rDZSABqKMzAPMb5ATWcvvdMljRzEh0LjBo=
 github.com/bytbox/go-pop3 v0.0.0-20120201222208-3046caf0763e/go.mod h1:alXX+s7a4cKaIprgjeEboqi4Tm7XR/HXEwUTxUV/ywU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/coreos/bbolt v0.0.0-20180318001526-af9db2027c98/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -172,8 +172,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -232,16 +230,16 @@ golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2 h1:T5DasATyLQfmbTpfEXx/IOL9vfjzW6up+ZDkmHvIf2s=
 golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=

main.go

@@ -1,9 +1,10 @@
 package main
 
 import (
-	"local/sandbox/contact/contact"
 	"log"
 	"regexp"
+
+	"gitea.inhome.blapointe.com/local-sandbox/contact"
 )
 
 func main() {
@@ -12,6 +13,7 @@ func main() {
 		IMAP:     config.EmailIMAP,
 		From:     config.EmailUser,
 		Password: config.EmailPass,
+		Limit:    config.EmailLimit,
 	}
 	emails, err := emailer.Read()
 	if err != nil {
@@ -33,7 +35,7 @@ func main() {
 				log.Printf("skipping match account antipattern %q vs %q", config.AccountsAntiPattern, transaction.Account)
 				continue
 			}
-			if _, err := config.Storage.Get(transaction.ID); err == nil {
+			if v, err := config.Storage.Get(transaction.ID); err == nil || string(v) == transaction.String() {
 				log.Println("skipping duplicate transaction:", transaction)
 			} else {
 				if err := Upload(config, transaction); err != nil {

scrape.go

@@ -15,6 +15,7 @@ type scraper interface {
 	scrape(*mail.Message) ([]*Transaction, error)
 }
 
+type fidelityScraper struct{}
 type bankOfAmericaScraper struct{}
 type chaseScraper struct{}
 type citiScraper struct{}
@@ -30,13 +31,16 @@ func Scrape(m *mail.Message, banks map[Bank]bool) ([]*Transaction, error) {
 
 func buildScraper(m *mail.Message, banks map[Bank]bool) (scraper, error) {
 	subject := fmt.Sprint(m.Header["Subject"])
-	if !containsAny(subject, "transaction", "report", "Transaction", "payment", "Payment") {
+	if !containsAny(subject, "transaction", "report", "Transaction", "payment", "Payment", "Deposit", "Withdrawal") {
 		return nil, errors.New("cannot build scraper for subject " + subject)
 	}
 	from := fmt.Sprint(m.Header["From"])
 	if strings.Contains(from, "Chase") && banks[Chase] {
 		return newChaseScraper(), nil
 	}
+	if strings.Contains(from, "Fidelity") && banks[Fidelity] {
+		return newFidelityScraper(), nil
+	}
 	if strings.Contains(from, "Bank of America") && banks[BankOfAmerica] {
 		return newBankOfAmericaScraper(), nil
 	}
@@ -49,6 +53,10 @@ func buildScraper(m *mail.Message, banks map[Bank]bool) (scraper, error) {
 	return nil, errors.New("unknown sender: " + from)
 }
 
+func newFidelityScraper() scraper {
+	return &fidelityScraper{}
+}
+
 func newBankOfAmericaScraper() scraper {
 	return &bankOfAmericaScraper{}
 }
@@ -210,6 +218,52 @@ func (c *uccuScraper) scrape(m *mail.Message) ([]*Transaction, error) {
 	return []*Transaction{transaction}, nil
 }
 
+func (c *fidelityScraper) scrape(m *mail.Message) ([]*Transaction, error) {
+	subject := fmt.Sprint(m.Header["Subject"])
+	if strings.Contains(subject, "Debit Withdrawal") {
+		return c.scrapeWithdrawal(m)
+	}
+	if strings.Contains(subject, "Deposit Received") {
+		return c.scrapeDeposit(m)
+	}
+	panic(nil)
+}
+
+func (c *fidelityScraper) scrapeDeposit(m *mail.Message) ([]*Transaction, error) {
+	b, err := ioutil.ReadAll(m.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	fidelAcc, _ := findSubstringBetween(b, "Account: XXXXX", "\n")
+
+	transaction := NewTransaction(
+		fmt.Sprintf("%s-%s", Fidelity, fidelAcc),
+		"?.??",
+		"misc",
+		fmt.Sprint(m.Header["Date"]),
+		Fidelity,
+	)
+	return []*Transaction{transaction}, nil
+}
+
+func (c *fidelityScraper) scrapeWithdrawal(m *mail.Message) ([]*Transaction, error) {
+	b, err := ioutil.ReadAll(m.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	amount, amountOk := findSubstringBetween(b, "in the amount of $", " ")
+	fidelAcc, fidelAccOk := findSubstringBetween(b, "For account ending in ", ":")
+	acc, accOk := findSubstringBetween(b, "in the amount of $"+amount+" by ", ".")
+
+	if amount == "" || acc == "" {
+		return nil, fmt.Errorf("no amount/account found: fidelAcc=%v,fidelAccOk=%v, acc=%v,accOk=%v, amount=%v,amountOk=%v", fidelAcc, fidelAccOk, acc, accOk, amount, amountOk)
+	}
+	transaction := NewTransaction(fmt.Sprintf("%s-%s", Fidelity, fidelAcc), amount, acc, fmt.Sprint(m.Header["Date"]), Fidelity)
+	return []*Transaction{transaction}, nil
+}
+
 func (c *bankOfAmericaScraper) scrape(m *mail.Message) ([]*Transaction, error) {
 	subject := fmt.Sprint(m.Header["Subject"])
 	if strings.Contains(subject, "Credit card transaction") {
@@ -227,8 +281,8 @@ func (c *bankOfAmericaScraper) scrapeCharge(m *mail.Message) ([]*Transaction, er
 		return nil, err
 	}
 
-	amount := c.findFloatAfter(b, "Amount: $")
+	amount := findFloatAfter(b, "Amount: $")
-	acc := string(c.findLineAfter(b, "Where: "))
+	acc := string(findLineAfter(b, "Where: "))
 
 	if amount == "" || acc == "" {
 		return nil, errors.New("no amount/account found")
@@ -242,7 +296,7 @@ func (c *bankOfAmericaScraper) scrapePayment(m *mail.Message) ([]*Transaction, e
 	if err != nil {
 		return nil, err
 	}
-	amount := "-" + c.findFloatAfter(b, "Payment: $")
+	amount := "-" + findFloatAfter(b, "Payment: $")
 	acc := "Payment"
 	if amount == "" || acc == "" {
 		return nil, errors.New("no amount/account found")
@@ -251,8 +305,20 @@ func (c *bankOfAmericaScraper) scrapePayment(m *mail.Message) ([]*Transaction, e
 	return []*Transaction{transaction}, nil
 }
 
-func (c *bankOfAmericaScraper) findFloatAfter(b []byte, prefix string) string {
+func findSubstringBetween(b []byte, prefix, suffix string) (string, bool) {
-	amount := string(c.findLineAfter(b, prefix))
+	byPre := bytes.Split(b, []byte(prefix))
+	if len(byPre) < 2 {
+		return "", false
+	}
+	bySuff := bytes.Split(byPre[1], []byte(suffix))
+	if len(bySuff) < 2 {
+		return "", false
+	}
+	return string(bySuff[0]), true
+}
+
+func findFloatAfter(b []byte, prefix string) string {
+	amount := string(findLineAfter(b, prefix))
 	words := strings.Split(amount, " ")
 	lastword := words[len(words)-1]
 	escapedfloat := strings.TrimPrefix(lastword, "$")
@@ -261,7 +327,7 @@ func (c *bankOfAmericaScraper) findFloatAfter(b []byte, prefix string) string {
 	return amount
 }
 
-func (c *bankOfAmericaScraper) findLineAfter(b []byte, prefix string) []byte {
+func findLineAfter(b []byte, prefix string) []byte {
 	for _, line := range bytes.Split(b, []byte("\n")) {
 		if bytes.HasPrefix(line, []byte(prefix)) {
 			return bytes.TrimSpace(bytes.TrimPrefix(line, []byte(prefix)))

scrape_test.go

@@ -179,6 +179,76 @@ func TestScrapeBofAPayment(t *testing.T) {
 	}
 }
 
+func TestScrapeFidelityDeposit(t *testing.T) {
+	b, err := ioutil.ReadFile("./testdata/fidelity.deposit.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	message := &mail.Message{
+		Header: map[string][]string{
+			"Subject": []string{"Fidelity Alerts: Deposit Received"},
+		},
+		Body: bytes.NewReader(b),
+	}
+	fidelity := &fidelityScraper{}
+
+	gots, err := fidelity.scrape(message)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(gots) != 1 {
+		t.Fatal(len(gots))
+	}
+	got := gots[0]
+	want := Transaction{
+		ID:      got.ID,
+		Bank:    Fidelity,
+		Amount:  "?.??",
+		Vendor:  "misc",
+		Date:    "[]",
+		Account: Fidelity.String() + "-5576",
+	}
+	if *got != want {
+		t.Fatalf("want:\n\t%+v, got\n\t%+v", want, *got)
+	}
+}
+
+func TestScrapeFidelityWithdrawal(t *testing.T) {
+	b, err := ioutil.ReadFile("./testdata/fidelity.withdrawal.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	message := &mail.Message{
+		Header: map[string][]string{
+			"Subject": []string{"Fidelity Alerts - Direct Debit Withdrawal"},
+		},
+		Body: bytes.NewReader(b),
+	}
+	fidelity := &fidelityScraper{}
+
+	gots, err := fidelity.scrape(message)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(gots) != 1 {
+		t.Fatal(len(gots))
+	}
+	got := gots[0]
+	want := Transaction{
+		ID:      got.ID,
+		Bank:    Fidelity,
+		Amount:  "1.00",
+		Vendor:  "CHASE CREDIT CRD",
+		Date:    "[]",
+		Account: Fidelity.String() + "-5576",
+	}
+	if *got != want {
+		t.Fatalf("want:\n\t%+v, got\n\t%+v", want, *got)
+	}
+}
+
 func TestScrapeBofACharge(t *testing.T) {
 	b, err := ioutil.ReadFile("./testdata/bofa.charge.txt")
 	if err != nil {

testdata/fidelity.deposit.txt

@@ -0,0 +1,29 @@
+
+Account: XXXXX5576
+
+A deposit to your account was received on 12/08/2022.
+
+		
+		
+Important information:
+Fidelity automatically emails certain alerts to customers who have provided
+an email address. You can see the terms which govern these alerts at
+https://www.fidelity.com/customer-service/alerts-agreement. If you would
+prefer to not receive these alerts, please change your preferences at
+https://scs.fidelity.com/customeronly/alerts.shtml.
+
+Review Fidelity's Terms of Use for Third Party Content and Research at
+https://www.fidelity.com/terms-of-use#Third.
+
+If your email has changed, please update your email address
+at https://alertable.fidelity.com/ftgw/alerts/GetUserDeliveryDevices to continue to
+receive your alerts.
+
+Fidelity Brokerage Services LLC, Member NYSE, SIPC
+
+EMAIL REF# 537048
+
+Copyright 2022 FMR LLC
+All rights reserved. Important Legal Information
+at http://www.fidelity.com/terms-of-use.
+

testdata/fidelity.withdrawal.txt

@@ -0,0 +1,49 @@
+
+For account ending in 5576:
+Money was withdrawn from your account through a direct debit in the amount of $1.00 by CHASE CREDIT CRD.
+
+If you authorized this transaction, no action is needed.
+ If you did not authorize this transaction, please contact us immediately at 800-343-3548.
+
+		
+		
+Important information:
+Fidelity automatically emails certain alerts to customers who have provided
+an email address. You can see the terms which govern these alerts at
+https://alertable.fidelity.com/alerts/help/agreement.html. If you would
+prefer to not receive these alerts, please change your preferences at
+https://scs.fidelity.com/customeronly/alerts.shtml.
+This new alerts service will not affect delivery of paper communications
+you are scheduled to receive.
+
+To stop receipt of alerts, modify your preferences on the Existing Alerts page
+at https://scs.fidelity.com/customeronly/fens.shtml, or temporarily
+stop/restart alerts at
+https://scs.fidelity.com/customeronly/fens_alertstatus.shtml.
+
+Fidelity offers access to a broader range of third-party research at
+http://personal.fidelity.com/research/stocks/content/stocksindex.shtml.
+Fidelity is not recommending or endorsing any third-party research by making it
+available to its customers or by notifying customers of its availability.
+
+Review Fidelity's Terms of Use for Third Party Content and Research at
+http://activequote.fidelity.com/rtrnews/terms.html.
+
+If your email has changed, please update your email address
+at https://scs.fidelity.com/customeronly/fens_profile.shtml to continue to
+receive your alerts.
+
+Read Fidelity's Commitment to Privacy
+at http://personal.fidelity.com/global/search/content/privacy.html.tvsr.
+
+Visit Fidelity's Home Page
+http://www.fidelity.com/
+
+Fidelity Brokerage Services LLC, Member NYSE, SIPC, 900 Salem Street, Smithfield, RI 02917
+
+EMAIL REF# 537048
+
+Copyright 2022 FMR LLC
+All rights reserved. Important Legal Information
+at http://personal.fidelity.com/misc/legal/legal.html.tvsr.
+

upload.go

@@ -1,27 +1,20 @@
 package main
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
-	"io/ioutil"
-	"local/oauth2"
-	"net/http"
-	"net/url"
 	"os"
-	"path"
 	"regexp"
 	"strconv"
-	"strings"
 	"time"
 
-	yaml "gopkg.in/yaml.v2"
+	"github.com/google/uuid"
 )
 
 func Upload(config Config, transaction *Transaction) error {
 	switch config.Uploader {
 	case UploaderTodo:
-		return uploadTodo(config, transaction)
+		panic("DEAD")
 	case UploaderLedger:
 		return uploadLedger(config, transaction)
 	case UploaderPTTodo:
@@ -31,67 +24,14 @@ func Upload(config Config, transaction *Transaction) error {
 	}
 }
 
-func uploadTodo(config Config, transaction *Transaction) error {
-	params := url.Values{
-		"list":  {config.TodoList},
-		"title": {transaction.Format()},
-		"tag":   {config.TodoTag},
-	}
-	req, err := http.NewRequest("POST", config.TodoAddr+"/ajax.php?newTask", strings.NewReader(params.Encode()))
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Cookie", oauth2.COOKIE+"="+config.TodoToken)
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		b, _ := ioutil.ReadAll(resp.Body)
-		return fmt.Errorf("bad status from todo: %v: %s", resp.StatusCode, b)
-	}
-	return nil
-}
-
 func uploadPTTodo(config Config, transaction *Transaction) error {
-	b, err := ioutil.ReadFile(config.TodoAddr)
+	f, err := os.Create(fmt.Sprintf("%s.todo.%s", config.TodoAddr, uuid.New().String()))
-	if os.IsNotExist(err) {
-		b = []byte("todo:\n")
-	} else if err != nil {
-		return err
-	} else if len(b) == 0 {
-		b = []byte("todo:\n")
-	}
-	f, err := ioutil.TempFile(path.Dir(config.TodoAddr), path.Base("."+config.TodoAddr))
 	if err != nil {
 		return err
 	}
 	defer f.Close()
-	sep := []byte{'\n'}
+	fmt.Fprintf(f, `- {"todo":%q, "tags":%q}%s`, transaction.Format(), config.TodoTag, "\n")
-	seek := []byte("todo:")
+	return f.Close()
-	for len(b) > 0 {
-		idx := bytes.Index(b, sep)
-		if idx == -1 {
-			idx = len(b) - 1
-		}
-		fmt.Fprintf(f, "%s\n", b[:idx])
-		if bytes.Equal(bytes.TrimSpace(b[:idx]), seek) {
-			fmt.Fprintf(f, `- {"todo":%q, "tags":%q}%s`, transaction.Format(), config.TodoTag, "\n")
-		}
-		b = b[idx+1:]
-	}
-	f.Close()
-	var v interface{}
-	b, err = ioutil.ReadFile(f.Name())
-	if err != nil {
-		return err
-	}
-	if err := yaml.Unmarshal(b, &v); err != nil {
-		return err
-	}
-	return os.Rename(f.Name(), config.TodoAddr)
 }
 
 func uploadLedger(config Config, transaction *Transaction) error {

upload_test.go

@@ -3,17 +3,30 @@ package main
 import (
 	"bytes"
 	"io/ioutil"
-	"local/storage"
 	"os"
 	"path"
+	"path/filepath"
 	"testing"
 
+	"gitea.inhome.blapointe.com/local/storage"
+
 	"github.com/google/uuid"
 )
 
 func TestUploadPTTodo(t *testing.T) {
 	addr := path.Join(t.TempDir(), "test.upload.pttodo")
 	config := Config{TodoAddr: addr, TodoTag: "expense"}
+	reset := func(t *testing.T) {
+		files, err := filepath.Glob(addr + "*")
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, f := range files {
+			if f != addr {
+				os.Remove(f)
+			}
+		}
+	}
 	xaction := func() *Transaction {
 		return &Transaction{
 			ID:     "id",
@@ -24,6 +37,7 @@ func TestUploadPTTodo(t *testing.T) {
 		}
 	}
 	t.Run("full file", func(t *testing.T) {
+		defer reset(t)
 		if err := ioutil.WriteFile(addr, []byte(`
 todo:
 - first
@@ -37,18 +51,18 @@ done: []
 		if err != nil {
 			t.Error(err)
 		}
-		b, err := ioutil.ReadFile(addr)
+		files, err := filepath.Glob(addr + ".todo.*")
 		if err != nil {
 			t.Error(err)
 		}
-		if bytes.Compare(bytes.TrimSpace(b), bytes.TrimSpace([]byte(`
+		if len(files) != 1 {
-todo:
+			t.Fatal(files)
-- {"todo":"(today) /UCCU: 1.23 @ vendor vendor", "tags":"expense"}
+		}
-- first
+		b, err := ioutil.ReadFile(files[0])
-- todo: second
+		if err != nil {
-scheduled: []
+			t.Error(err)
-done: []
+		}
-      `))) != 0 {
+		if bytes.Compare(bytes.TrimSpace(b), bytes.TrimSpace([]byte(`- {"todo":"(today) /UCCU: 1.23 @ vendor vendor", "tags":"expense"}`))) != 0 {
 			t.Errorf("full file came out wrong: got %s", b)
 		}
 		if !bytes.Contains(b, []byte(xaction().Format())) {
@@ -57,12 +71,20 @@ done: []
 		t.Logf("%s", b)
 	})
 	t.Run("no file", func(t *testing.T) {
+		defer reset(t)
 		os.Remove(addr)
 		err := uploadPTTodo(config, xaction())
 		if err != nil {
 			t.Error(err)
 		}
-		b, err := ioutil.ReadFile(addr)
+		files, err := filepath.Glob(addr + ".todo.*")
+		if err != nil {
+			t.Error(err)
+		}
+		if len(files) != 1 {
+			t.Fatal(files)
+		}
+		b, err := ioutil.ReadFile(files[0])
 		if err != nil {
 			t.Error(err)
 		}
@@ -72,6 +94,7 @@ done: []
 		t.Logf("%s", b)
 	})
 	t.Run("empty file", func(t *testing.T) {
+		defer reset(t)
 		if err := ioutil.WriteFile(addr, []byte{}, os.ModePerm); err != nil {
 			t.Fatal(err)
 		}
@@ -79,7 +102,14 @@ done: []
 		if err != nil {
 			t.Error(err)
 		}
-		b, err := ioutil.ReadFile(addr)
+		files, err := filepath.Glob(addr + ".todo.*")
+		if err != nil {
+			t.Error(err)
+		}
+		if len(files) != 1 {
+			t.Fatal(files)
+		}
+		b, err := ioutil.ReadFile(files[0])
 		if err != nil {
 			t.Error(err)
 		}