From fe24c6e2371b903a99e55f82d6df32b35f537489 Mon Sep 17 00:00:00 2001 From: Bel LaPointe Date: Mon, 18 May 2020 15:02:50 -0600 Subject: [PATCH] dunno --- .DS_Store | Bin 0 -> 6148 bytes .gitattributes | 2 + .gitignore | 10 ++ 2do | 15 +++ client/README | 16 +++ client/main.go | 146 +++++++++++++++++++++++++++ client/remote/remote.go | 95 ++++++++++++++++++ client/remote/remote_test.go | 32 ++++++ server/api.go | 139 ++++++++++++++++++++++++++ server/cert.pem | 18 ++++ server/entrypoint.sh | 24 +++++ server/httpr.go | 60 ++++++++++++ server/key.pem | 27 +++++ server/main.go | 41 ++++++++ server/start.minio | 54 ++++++++++ server/storage.go | 185 +++++++++++++++++++++++++++++++++++ 16 files changed, 864 insertions(+) create mode 100755 .DS_Store create mode 100755 .gitattributes create mode 100755 .gitignore create mode 100755 2do create mode 100755 client/README create mode 100755 client/main.go create mode 100755 client/remote/remote.go create mode 100755 client/remote/remote_test.go create mode 100755 server/api.go create mode 100755 server/cert.pem create mode 100755 server/entrypoint.sh create mode 100755 server/httpr.go create mode 100755 server/key.pem create mode 100755 server/main.go create mode 100755 server/start.minio create mode 100755 server/storage.go diff --git a/.DS_Store b/.DS_Store new file mode 100755 index 0000000000000000000000000000000000000000..ef82ecf21faacd9ab6b959ebf1f809df197588c8 GIT binary patch literal 6148 zcmeHKOKQU~5S>X)F?8c)mbyZ3AcA`WUm%c%5-={5HfyhPt{$y#J{#BVF4=@PFnZEx zo`l}Q;}H?veZ4-5bRyEi4dr6P+HBu^Wt)sB5RNk@`8*FF)A&BUt+M|P823>QGRjGg zzxcLCqXJZb3Qz$mKn1R=K-Smx>AJ`AG%7#^?mz+iJ`}iNO`HS$(}BTT0APc#8|L0i z0E-2HHE|9^1g1d+2351g(4Zq;GOs4ifk79|=0o#l%??HVcAQ^4U9<*rqyki6slY=l zJFEXM@PGRMC5bC4Kn3nf0d4xB-{DDFTUU>>T3g_IxaEAq%`kTg1~11zFUMF|Io^6w biGvE;K4|YXyD)bAA=5 literal 0 HcmV?d00001 diff --git a/.gitattributes b/.gitattributes new file mode 100755 index 0000000..007448d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +vendor/**/* -diff +vendor/vendor.json diff diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..92995f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +alt* +*.db +*.log +watchman +*.swp +*.swo +testdir +server/server +client/client +*/vendor/ diff --git a/2do b/2do new file mode 100755 index 0000000..25044c1 --- /dev/null +++ b/2do @@ -0,0 +1,15 @@ +server + allow batch pub key query + signal clients on file change so they can autopull +client + push with async encrypted + re-uses pub/pri keys + push without $HOME/prefix + push to all pub keys + auto-pull + +done + server + stores with targetID + client + push to specific pub key diff --git a/client/README b/client/README new file mode 100755 index 0000000..f5a02b4 --- /dev/null +++ b/client/README @@ -0,0 +1,16 @@ +* watch for file changes + https://github.com/fsnotify/fsnotify + https://medium.com/@skdomino/watch-this-file-watching-in-go-5b5a247cf71f + +* fault tolerant + https://github.com/klauspost/reedsolomon + +* minio for backup to GCP/AWS/Other + or just drive + pref minio for fun + +* encryption somehow + allow encrypt/decrypt anywhere + can restore with little to nothing + local/encrypter can encrypt/decrypt given access to pub/pri keys + symmetrical encrypt? diff --git a/client/main.go b/client/main.go new file mode 100755 index 0000000..574c857 --- /dev/null +++ b/client/main.go @@ -0,0 +1,146 @@ +package main + +import ( + "flag" + "fmt" + "local/watchman/client/remote" + "local/watchman/client/watchman" + "log" + "os" + "os/signal" +) + +func envOrDefault(key, alt string) string { + if v := os.Getenv(key); v != "" { + return v + } + log.Printf("ENV variable %q not set, defaulting to %q", key, alt) + return alt +} + +var defaultKey = envOrDefault("WATCHMANKEY", "key") +var defaultID = envOrDefault("WATCHMANID", "ID") + +type empty struct{} + +func catchInterrupt() chan os.Signal { + stop := make(chan os.Signal) + signal.Notify(stop, os.Interrupt) + return stop +} + +func main() { + stopWatch := make(chan empty) + fnames := make(chan string) + packs := make(chan *watchman.Package) + + pullPath := flag.String("path", "", "path to pull") + flag.Parse() + if *pullPath != "" { + go packPuller(packs, stopWatch) + packs <- &watchman.Package{Path: *pullPath} + stopWatch <- empty{} + return + } + + log.Print("Watching for changes...") + go watchFiles("testdir", fnames, stopWatch) + + log.Print("Packing changes...") + go filePacker(packs, fnames, stopWatch) + + log.Print("Pushing changes...") + go packPusher(packs, stopWatch) + + log.Print("Waiting for signal...") + <-catchInterrupt() + for i := 0; i < 3; i++ { + stopWatch <- empty{} + } + + fmt.Println() + log.Print("Exiting") +} + +func packPuller(packs <-chan *watchman.Package, stop <-chan empty) { + rem, err := remote.New() + if err != nil { + log.Fatal("can't connect to remote:", err) + } + exiting := false + for !exiting { + select { + case pack := <-packs: + pack.ID = defaultID + backpack, err := rem.Pull(pack) + if err != nil { + log.Print("error pulling:", err) + break + } + if backpack == nil { + break + } + decrypted, err := watchman.UnpackNDecrypt(backpack, defaultKey) + if err != nil { + log.Print("error unpacking and decrypting:", err) + break + } + log.Print(string(decrypted)) + case <-stop: + exiting = true + } + } +} + +func packPusher(packs <-chan *watchman.Package, stop <-chan empty) { + rem, err := remote.New() + if err != nil { + log.Fatal("can't connect to remote:", err) + } + exiting := false + for !exiting { + select { + case pack := <-packs: + if err := rem.Push(pack); err != nil { + log.Print("error pushing:", err) + } + case <-stop: + exiting = true + } + } +} + +func filePacker(packs chan<- *watchman.Package, fnames <-chan string, stop <-chan empty) { + exiting := false + for !exiting { + select { + case fname := <-fnames: + pack, err := watchman.EncryptNPack(fname, defaultKey, defaultID) + if err != nil { + log.Printf("err packingNencrypting file %q: %v", fname, err) + break + } + packs <- pack + case <-stop: + exiting = true + } + } +} + +func watchFiles(path string, fnames chan<- string, stop <-chan empty) { + watcher := watchman.NewWatcher(path) + if err := watcher.Start(); err != nil { + panic(err) + } + exiting := false + for !exiting { + select { + case fname := <-watcher.Files: + fnames <- fname + case <-stop: + exiting = true + watcher.Stop() + return + } + } +} diff --git a/client/remote/remote.go b/client/remote/remote.go new file mode 100755 index 0000000..a8e1e75 --- /dev/null +++ b/client/remote/remote.go @@ -0,0 +1,95 @@ +package remote + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "local/encryptor" + "local/system/sysconf" + "local/watchman/client/watchman" + "net/http" + "strings" +) + +type Remote struct { + enc encryptor.Encryptor + client *http.Client + url string +} + +func New() (*Remote, error) { + r := &Remote{ + url: "https://localhost:" + strings.Split(sysconf.Get("watchmans").Port, ",")[1], + client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + }, + enc: encryptor.NewEncryptor("", ""), + } + body, err := r.request("getpub", sysconf.Get("watchmans").Name) + if err != nil { + return nil, errors.New("can't contact watchmans") + } + r.enc.SetPublic(string(body)) + return r, nil +} + +func (r *Remote) request(path string, body string) ([]byte, error) { + req, err := http.NewRequest("GET", fmt.Sprintf("%v/%v", r.url, path), bytes.NewBuffer([]byte(body))) + if err != nil { + return nil, err + } + resp, err := r.client.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, errors.New("error when accessing " + path + ": http status " + resp.Status) + } + defer resp.Body.Close() + return ioutil.ReadAll(resp.Body) +} + +func (r *Remote) Push(pack *watchman.Package) error { + b, err := json.Marshal(pack) + if err != nil { + return err + } + if len(b) == 0 { + return nil + } + encrypted := r.enc.Encrypt(string(b)) + _, err = r.request("push", encrypted) + return err +} + +func (r *Remote) Pull(pack *watchman.Package) (*watchman.Package, error) { + transactionKey := encryptor.NewEncryptor("", "").GetPublic() + transactionKey = transactionKey[len(transactionKey)/2:] + transactionKey = transactionKey[:20] + pack.Packs = [][]byte{[]byte(encryptor.NewEncryptor("", "").GetPublic())} + dec := encryptor.NewEncryptor("", "") + dec.SetSymmetric(string(pack.Packs[0])) + b, err := json.Marshal(pack) + if err != nil { + return nil, err + } + if len(b) == 0 { + return nil, nil + } + encrypted := r.enc.Encrypt(string(b)) + var ret watchman.Package + b, err = r.request("pull", encrypted) + if err != nil { + return nil, err + } + b = []byte(dec.Decrypt(string(b))) + if err := json.Unmarshal(b, &ret); err != nil { + return nil, err + } + return &ret, nil +} diff --git a/client/remote/remote_test.go b/client/remote/remote_test.go new file mode 100755 index 0000000..10c1444 --- /dev/null +++ b/client/remote/remote_test.go @@ -0,0 +1,32 @@ +package remote + +import ( + "local/watchman/client/watchman" + "testing" +) + +func TestPush(t *testing.T) { + r := New() + cases := []struct { + pack *watchman.Package + }{} + for _, c := range cases { + err := r.Push(c.pack) + if err != nil { + t.Fatalf("error pushing test pack: %v", err) + } + } +} + +func TestPull(t *testing.T) { + r := New() + cases := []struct { + name string + }{} + for _, c := range cases { + _, err := r.Pull(c.name) + if err != nil { + t.Fatalf("error pulling test pack: %v", err) + } + } +} diff --git a/server/api.go b/server/api.go new file mode 100755 index 0000000..e4633f1 --- /dev/null +++ b/server/api.go @@ -0,0 +1,139 @@ +package main + +import ( + "encoding/json" + "local/encryptor" + "local/system/sysconf" + "local/watchman/client/watchman" + "log" + "net/http" +) + +type API struct { + conf sysconf.Conf + enc encryptor.Encryptor + storage *Storage + junk map[string]string + nextJunk string +} + +func newAPI(conf sysconf.Conf) (*API, error) { + store, err := newStorage() + if err != nil { + return nil, err + } + enc := encryptor.NewEncryptor("", "") + if err := store.set(keyBucket, serviceName, enc.GetPublic()); err != nil { + return nil, err + } + return &API{ + storage: store, + conf: conf, + enc: enc, + junk: make(map[string]string), + nextJunk: encryptor.NewEncryptor("", "").GetPublic(), + }, nil +} + +func (api *API) ServeHTTP(w http.ResponseWriter, r *http.Request) { + httpr := &HTTPRequest{w: w, r: r} + action := httpr.PathAdvance() + if action == "getpub" { + api.pubKeyGet(httpr) + return + } + if err := httpr.readBody(api.enc); err != nil { + return + } + switch action { + case "setpub": + api.pubKeySet(httpr) + case "push": + api.push(httpr) + case "pull": + api.pull(httpr) + default: + w.WriteHeader(http.StatusNotFound) + } +} +func (api *API) pubKeyGet(httpr *HTTPRequest) { + body, err := httpr.limitReader() + if err != nil { + httpr.result(http.StatusRequestEntityTooLarge, err, nil) + return + } + if len(body) == 0 { + httpr.result(http.StatusBadRequest, nil, nil) + return + } + // if asks for server pub, then return server pub + if string(body) == serviceName { + httpr.result(http.StatusOK, nil, api.enc.GetPublic()) + return + } + // else if encrypted with server's pub, then return any + if decrypted := api.enc.Decrypt(string(body)); decrypted != "" { + if pub, err := api.storage.get(keyBucket, decrypted); err == nil { + httpr.result(http.StatusOK, nil, pub) + } + httpr.result(http.StatusNotFound, nil, nil) + return + } + // else return garbage + if v, ok := api.junk[string(body)]; ok { + httpr.result(secretError, nil, v) + } else { + api.junk[string(body)] = api.nextJunk + httpr.result(secretError, nil, api.junk[string(body)]) + go func() { + api.nextJunk = encryptor.NewEncryptor("", "").GetPublic() + }() + } +} + +func (api *API) pubKeySet(httpr *HTTPRequest) { + var setRequest struct { + Key string `json:"key"` + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(httpr.body), &setRequest); err != nil { + httpr.result(http.StatusBadRequest, nil, nil) + return + } + api.storage.set(keyBucket, setRequest.ID, setRequest.Key) +} + +func (api *API) push(httpr *HTTPRequest) { + var pack watchman.Package + if err := json.Unmarshal([]byte(httpr.body), &pack); err != nil { + httpr.result(http.StatusBadRequest, err, nil) + return + } + if err := api.storage.set("objects", storePath(pack), httpr.body); err != nil { + httpr.result(http.StatusInternalServerError, err, nil) + return + } + httpr.result(http.StatusOK, nil, nil) +} + +func (api *API) pull(httpr *HTTPRequest) { + var pack watchman.Package + if err := json.Unmarshal([]byte(httpr.body), &pack); err != nil { + httpr.result(http.StatusBadRequest, err, nil) + return + } + b, err := api.storage.get("objects", storePath(pack)) + if err != nil { + httpr.result(http.StatusNotFound, err, nil) + return + } + enc := encryptor.NewEncryptor("", "") + enc.SetSymmetric(string(pack.Packs[0])) + encrypted := enc.Encrypt(string(b)) + log.Print("returning content", string(b)) + httpr.result(http.StatusOK, nil, encrypted) +} + +func storePath(pack watchman.Package) string { + return pack.ID + "_" + pack.Path +} diff --git a/server/cert.pem b/server/cert.pem new file mode 100755 index 0000000..26adbb9 --- /dev/null +++ b/server/cert.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC9zCCAd+gAwIBAgIQcgF95XkV06ypg+WfUhEegTANBgkqhkiG9w0BAQsFADAS +MRAwDgYDVQQKEwdBY21lIENvMB4XDTE4MDgxMDEzNTQyNVoXDTE5MDgxMDEzNTQy +NVowEjEQMA4GA1UEChMHQWNtZSBDbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBANQTRy/WN7m5H/E+T5eBo8UfFrTgMq1DK0dE9vBSs5yNgxb8qlXUJXsl +gTMOdBeijP4g47EMUMy2lOgAqeY8aWYvkq9hahnew7tpdKNd94+PKhPAN8NMbMH2 +MHSCUN9MZq+cj33ieGstf51z5ANhb9aP0HLbU9o549od/5QLOdklfopmwz9DGY3b +K2kzmBrH7n26CPmdxjKaemW6IiZU4SJBhCf3A/371IfZMk6EHZhZ+5TTl2mXyV/v +QBqZHGh5+nTccCVnHIRUrRgA0fqrxVuzgU/3EoHDihC60I0EynT+bFWp7TSsrNWX +K4ibFQg8YbTcyqszNWm3BlnM6ThF5G8CAwEAAaNJMEcwDgYDVR0PAQH/BAQDAgWg +MBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwEgYDVR0RBAswCYIH +Y2hpcmFtZTANBgkqhkiG9w0BAQsFAAOCAQEAlhOgt0biHgqiBg4V7cEcwfGrSwd4 +l4rT1HkmrTG4AKfPYKDMd7KcBOhkNMq89nQkQ3cZBM8s8JIzkpK4nqYCi0wIAAEf +QUJyCH9BwCTBZO7BIl3NTwQnx8yqmLesRnpCJo/hrbO8gZANO+1BCCov7Vl0jblE +4jbyP33xPUpTi7LPqoNSQVIaEoRj1B7BxViyYw2x5cpL6J52IKpSAnOPFqgeQxzz +CwFGuhPERZ8H0Fr9EfcxJzY3PFxyOHnsPizrQVFfrGkSZftIjGuwELhs/qJ4FAJ3 +oqLEXfOcjoJw/ebd5v+c0eQoNlmLeewWlE0p/zVzui2YVNOR0C4uqnM+Fg== +-----END CERTIFICATE----- diff --git a/server/entrypoint.sh b/server/entrypoint.sh new file mode 100755 index 0000000..fb92715 --- /dev/null +++ b/server/entrypoint.sh @@ -0,0 +1,24 @@ +#! /bin/bash + +set -e + +function cleanDocker() { + echo "not killing" + #echo Killing $CONTAINER_NAME + #docker kill $CONTAINER_NAME +} +trap cleanDocker EXIT + +cd "$(dirname ${BASH_SOURCE[0]})" + +if [ ! -f ./key.pem ] || [ ! -f ./cert.pem ]; then + rm -f ./key.pem ./cert.pem + go run ./vendor/crypto/tls/generate_cert.go --host chirame +fi + +go build + +source ./start.minio + +./$(basename $(pwd)) + diff --git a/server/httpr.go b/server/httpr.go new file mode 100755 index 0000000..f6e5d22 --- /dev/null +++ b/server/httpr.go @@ -0,0 +1,60 @@ +package main + +import ( + "errors" + "fmt" + "io/ioutil" + "local/encryptor" + "log" + "net/http" + "path" + "strings" +) + +type HTTPRequest struct { + w http.ResponseWriter + r *http.Request + body string +} + +func (httpr *HTTPRequest) PathAdvance() string { + p := path.Clean("/" + httpr.r.URL.Path) + i := strings.Index(p[1:], "/") + 1 + if i <= 0 { + httpr.r.URL.Path = "/" + return p[1:] + } + httpr.r.URL.Path = p[i:] + return p[1:i] +} + +func (httpr *HTTPRequest) result(status int, err error, body interface{}) { + if err != nil { + log.Print(err) + } + httpr.w.WriteHeader(status) + if body != nil { + fmt.Fprintln(httpr.w, body) + } +} + +func (httpr *HTTPRequest) limitReader(n ...int) ([]byte, error) { + if len(n) < 1 { + return ioutil.ReadAll(http.MaxBytesReader(nil, httpr.r.Body, int64(2048))) + } + return ioutil.ReadAll(http.MaxBytesReader(nil, httpr.r.Body, int64(n[0]))) +} + +func (httpr *HTTPRequest) readBody(enc encryptor.Encryptor) error { + body, err := httpr.limitReader() + if err != nil { + httpr.result(http.StatusRequestEntityTooLarge, err, nil) + return err + } + httpr.body = enc.Decrypt(string(body)) + if httpr.body == "" { + httpr.result(secretError, nil, nil) + return errors.New("wrong pub key") + } + return nil +} diff --git a/server/key.pem b/server/key.pem new file mode 100755 index 0000000..109a3ef --- /dev/null +++ b/server/key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA1BNHL9Y3ubkf8T5Pl4GjxR8WtOAyrUMrR0T28FKznI2DFvyq +VdQleyWBMw50F6KM/iDjsQxQzLaU6ACp5jxpZi+Sr2FqGd7Du2l0o133j48qE8A3 +w0xswfYwdIJQ30xmr5yPfeJ4ay1/nXPkA2Fv1o/QcttT2jnj2h3/lAs52SV+imbD +P0MZjdsraTOYGsfufboI+Z3GMpp6ZboiJlThIkGEJ/cD/fvUh9kyToQdmFn7lNOX +aZfJX+9AGpkcaHn6dNxwJWcchFStGADR+qvFW7OBT/cSgcOKELrQjQTKdP5sVant +NKys1ZcriJsVCDxhtNzKqzM1abcGWczpOEXkbwIDAQABAoIBAF8EgCmTrgX9Rq18 +wIZeJDUmd7L0nF/6r0JQSN9l/mlPEgPTkrG/ykdBh4CLokIQp2EY9UsW/ICr8U19 +NqIcQRDykaMYX0RWBZZLamHjsQ5WE4Ej5xgOfs/scMtSs89IWN2npLa/KDrE+N9f +1DIbjtMwPjGnyQsGUusf86mt2e5KyG6wljBC4QfxyDpfkj4VD4Gx45poeeWaQEhp +vHWSWwbbTAGwkgriLh68Y/fxg/NNXTgx0O03oukZR7JM2zAKlNFegcsiPsVa4KbG +tJeszezIyLIHW+CH42jxHPqwgITdEWmIMvvwi1faoE27Td8NwU1IjeWIHRvW66o7 +w6ZwQyECgYEA4/v6fS6MIMttRhrodGIsCHewN+iesVy/uzuVV+Im1w223ziMLanR +jBu458HEh3QC84T/ZziLtq1te9B+QWT8lCZ75Hje5QpPBgj4w2+AqueHeMPCOGZ2 +5hnZzWBhTAaRmY7yGDcXQ1V01dAbxWz6D6HYHktg0f9VhbmNfWuladECgYEA7iLU +DO9AsdGnQMCaWZzoZdjUZ6GrdcQ3O60B71EsBSt+lEHGik8AfJF0QKSJbanr4It/ +9UxSgiFTVYQ1ZiKJzGrJqZ/+n3TYrIuvUTFh29UtE07RwAfb+O+YMBW47AldRqOn +5hzGDwIZs8N38oQzts6WnmOffAK+jNSZK31/uj8CgYEAzrHP6xh9cOod+xZAM+wh +kde5iya6YYD+T2j4wEHIBudnKb/hzzCMS+OCY5PYcxnp5xBoYhPxD0Dy5vMi1HUT +TdoKLxyqsKsE9CrEJqP72nao6wNIHcw/9ePwBHRiIgQ+kyL3OJ8R1zkuAP95fieM +GwoXn3elox5EUkXlEpW61VECgYBOqCRgjVpSIczb34JcHS7KDT/DZywqPwB7bp7X +/HjM0FwD/mHk50li2+yJOY/HMDwgNBO041vRbc6HzZ6RuNDJO3CW3akN5Ft5Sr5C +1Evdf+Feokc35aCr7f/XyiUFmeY8YewgXtqwtGHm9aaV7ULjnAM2F/Pi00k7XTGm +otJgowKBgQDCxx0oqEvq9rJvRzGSlQyX2y0Zdw8clZKRVBeHGirCg/0iRzn9cQ2x +Dn5r2XSfPUpMw/MEO2175PwTqmNtzR7WfPye2FUdbn4zaS9mHqcFz2774Me/NTNf +V404gY4ZhI3k2Jie0cavFBI1FDI7pQVf2roYnKgPhF4xUHvLcj87FA== +-----END RSA PRIVATE KEY----- diff --git a/server/main.go b/server/main.go new file mode 100755 index 0000000..9097282 --- /dev/null +++ b/server/main.go @@ -0,0 +1,41 @@ +package main + +import ( + "local/system/sysconf" + "log" + "net/http" + "strings" +) + +var serviceName = sysconf.Get("watchmans").Name + +const secretError = http.StatusOK + +func redirHandle(w http.ResponseWriter, r *http.Request) { + host := strings.Split(r.Host, ":")[0] + ":" + strings.Split(sysconf.Get(serviceName).Port, ",")[0] + target := "https://" + host + r.URL.Path + if len(r.URL.RawQuery) > 0 { + target += "?" + r.URL.RawQuery + } + log.Print(target) + http.Redirect(w, r, target, http.StatusTemporaryRedirect) +} + +func main() { + conf := sysconf.Get(serviceName) + ports := strings.Split(conf.Port, ",") + if len(ports) < 2 { + panic("not enough ports in sysconf") + } + log.Print("making new API") + api, err := newAPI(conf) + if err != nil { + log.Fatal(err) + } + log.Printf("Listening for http on %v", ports[0]) + go func() { + log.Fatal(http.ListenAndServe(":"+ports[0], http.HandlerFunc(redirHandle))) + }() + log.Printf("Listening for https on %v", ports[1]) + log.Fatal(http.ListenAndServeTLS(":"+ports[1], "./cert.pem", "./key.pem", api)) +} diff --git a/server/start.minio b/server/start.minio new file mode 100755 index 0000000..49dd261 --- /dev/null +++ b/server/start.minio @@ -0,0 +1,54 @@ +#! /bin/bash + +set -u +set -e + +function start_minio() { + cd "$(dirname ${BASH_SOURCE[0]})" + + AWS_ACCESS="${user:-}" + AWS_SECRET="${pass:-}" + AWS_PORT="${AWS_PORT:-11000}" + AWS_REGION="local" + CONTAINER_NAME="watchman-minio" + + if [ "$(docker ps | grep $CONTAINER_NAME)" == "" ]; then + echo "Starting minio..." + docker run -d --rm \ + -p $AWS_PORT:$AWS_PORT \ + -e MINIO_ACCESS_KEY="$AWS_ACCESS" \ + -e MINIO_SECRET_KEY="$AWS_SECRET" \ + -e MINIO_REGION="$AWS_REGION" \ + --name $CONTAINER_NAME \ + minio/minio \ + server --address ":$AWS_PORT" /data + sleep 10 + else + echo "minio already running" + fi + keys="$(docker exec $CONTAINER_NAME cat /root/.minio/config.json | grep Key)" + STORE_ID="${keys#*: \"}" + STORE_ID="${STORE_ID%%\"*}" + STORE_LOC="localhost:$AWS_PORT" + STORE_SECRET="${keys##*secretKey\": \"}" + STORE_SECRET="${STORE_SECRET%%\"*}" + export STORE_ID="$STORE_ID" + export STORE_SECRET="$STORE_SECRET" + export STORE_LOC="$STORE_LOC" + export STORE_REGION="$AWS_REGION" + echo "Waiting for minio..." + skip=5 + for((i=0; i<60; i+=skip)); do + printf "\tcheck %d..." $((i/skip)) + if [ "$(docker ps | grep $CONTAINER_NAME | grep "(healthy)")" != "" ]; then + echo "" + return + fi + sleep $skip + done + echo "\nMinio never healthy" + docker logs $CONTAINER_NAME + exit 1 +} + +start_minio diff --git a/server/storage.go b/server/storage.go new file mode 100755 index 0000000..e14c347 --- /dev/null +++ b/server/storage.go @@ -0,0 +1,185 @@ +package main + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "strings" + "time" + + minio "github.com/minio/minio-go" +) + +var nilLocation = os.Getenv("STORE_REGION") + +const keyBucket = "key" + +var bucketNames = []string{} + +type Storage struct { + client *minio.Client +} + +func assertEnv(key string) (string, error) { + value := os.Getenv(key) + if value == "" { + return "", errors.New(key + " not set") + } + return value, nil +} + +func newStorage() (*Storage, error) { + var id, loc, secret string + var err error + if id, err = assertEnv("STORE_ID"); err != nil { + return nil, err + } + if secret, err = assertEnv("STORE_SECRET"); err != nil { + return nil, err + } + if loc, err = assertEnv("STORE_LOC"); err != nil { + return nil, err + } + if _, err = assertEnv("STORE_REGION"); err != nil { + return nil, err + } + client, err := minio.New(loc, id, secret, false) + if err != nil { + return nil, err + } + if err := makeInitialBuckets(client); err != nil { + return nil, err + } + return &Storage{ + client: client, + }, err +} + +func makeInitialBuckets(client *minio.Client) error { + buckets, err := client.ListBuckets() + if err != nil { + return err + } + keysFound := false + for _, bucketInfo := range buckets { + if bucketInfo.Name == keyBucket { + keysFound = true + } + bucketNames = append(bucketNames, bucketInfo.Name) + } + if !keysFound { + bucketNames = append(bucketNames, keyBucket) + } + for _, bucketName := range bucketNames { + if err := makeMinioBucket(client, bucketName); err != nil { + return err + } + } + return nil +} + +func makeMinioBucket(client *minio.Client, bucketName string) error { + log.Print("Making bucket", bucketName, "...") + if err := client.MakeBucket(bucketName, nilLocation); err == nil { + log.Print("...Made", bucketName) + return nil + } + exists, err := client.BucketExists(bucketName) + if err == nil && exists { + log.Print("...Exists", bucketName) + return nil + } + log.Print("...Can't make bucket", bucketName, err) + return err +} + +func (s *Storage) makeBucket(bucket string) error { + if bucket == keyBucket { + return errors.New("reserved bucket name") + } + err := makeMinioBucket(s.client, bucket) + if err == nil { + bucketNames = append(bucketNames, bucket) + } + return err +} + +func (s *Storage) toGetKey(fname, id string) string { + return fmt.Sprintf( + "%v_%v", + id, + strings.Split(fname, ".")[0], + ) +} + +func (s *Storage) toSetKey(fname, id string) string { + return fmt.Sprintf( + "%v_%v.%v", + s.toGetKey(fname, id), + time.Now().UnixNano(), + strings.Split(fname, ".")[1], + ) +} + +func (s *Storage) set(bucket, key, value string) error { + key = encodeKey(key) + log.Print("server setting in minio") + if !validBucket(bucket) { + log.Print("making bucket", bucket) + if err := s.makeBucket(bucket); err != nil { + log.Print("cantmake bucket", bucket) + return err + } + } + log.Print("putting", key) + _, err := s.client.PutObject(bucket, key, bytes.NewBuffer([]byte(value)), int64(len(value)), minio.PutObjectOptions{}) + log.Print("put err:", err) + return err +} + +func (s *Storage) get(bucket, key string) (string, error) { + key = encodeKey(key) + if !validBucket(bucket) { + return "", errors.New("unknown bucket") + } + done := make(chan struct{}) + defer close(done) + lastKey := "" + for obj := range s.client.ListObjectsV2(bucket, key, false, done) { + if lastKey == "" { + lastKey = obj.Key + } + if obj.Key > lastKey { + lastKey = obj.Key + } + } + if lastKey == "" { + return "", errors.New("object not found") + } + obj, err := s.client.GetObject(bucket, lastKey, minio.GetObjectOptions{}) + if err != nil { + return "", err + } + body, err := ioutil.ReadAll(obj) + return string(body), err +} + +func validBucket(bucket string) bool { + for i := range bucketNames { + if bucketNames[i] == bucket { + return true + } + } + return false +} + +func encodeKey(key string) string { + return strings.Replace(key, "/", "`", -1) +} + +func decodeKey(key string) string { + return strings.Replace(key, "`", "/", -1) +}