newline battles continue

vendor/github.com/ncw/rclone/fs/rc/internal.go

@@ -0,0 +1,131 @@
+// Define the internal rc functions
+
+package rc
+
+import (
+	"os"
+	"runtime"
+
+	"github.com/pkg/errors"
+)
+
+func init() {
+	Add(Call{
+		Path:  "rc/noop",
+		Fn:    rcNoop,
+		Title: "Echo the input to the output parameters",
+		Help: `
+This echoes the input parameters to the output parameters for testing
+purposes.  It can be used to check that rclone is still alive and to
+check that parameter passing is working properly.`,
+	})
+	Add(Call{
+		Path:  "rc/error",
+		Fn:    rcError,
+		Title: "This returns an error",
+		Help: `
+This returns an error with the input as part of its error string.
+Useful for testing error handling.`,
+	})
+	Add(Call{
+		Path:  "rc/list",
+		Fn:    rcList,
+		Title: "List all the registered remote control commands",
+		Help: `
+This lists all the registered remote control commands as a JSON map in
+the commands response.`,
+	})
+	Add(Call{
+		Path:  "core/pid",
+		Fn:    rcPid,
+		Title: "Return PID of current process",
+		Help: `
+This returns PID of current process.
+Useful for stopping rclone process.`,
+	})
+	Add(Call{
+		Path:  "core/memstats",
+		Fn:    rcMemStats,
+		Title: "Returns the memory statistics",
+		Help: `
+This returns the memory statistics of the running program.  What the values mean
+are explained in the go docs: https://golang.org/pkg/runtime/#MemStats
+
+The most interesting values for most people are:
+
+* HeapAlloc: This is the amount of memory rclone is actually using
+* HeapSys: This is the amount of memory rclone has obtained from the OS
+* Sys: this is the total amount of memory requested from the OS
+  * It is virtual memory so may include unused memory
+`,
+	})
+	Add(Call{
+		Path:  "core/gc",
+		Fn:    rcGc,
+		Title: "Runs a garbage collection.",
+		Help: `
+This tells the go runtime to do a garbage collection run.  It isn't
+necessary to call this normally, but it can be useful for debugging
+memory problems.
+`,
+	})
+}
+
+// Echo the input to the ouput parameters
+func rcNoop(in Params) (out Params, err error) {
+	return in, nil
+}
+
+// Return an error regardless
+func rcError(in Params) (out Params, err error) {
+	return nil, errors.Errorf("arbitrary error on input %+v", in)
+}
+
+// List the registered commands
+func rcList(in Params) (out Params, err error) {
+	out = make(Params)
+	out["commands"] = registry.list()
+	return out, nil
+}
+
+// Return PID of current process
+func rcPid(in Params) (out Params, err error) {
+	out = make(Params)
+	out["pid"] = os.Getpid()
+	return out, nil
+}
+
+// Return the memory statistics
+func rcMemStats(in Params) (out Params, err error) {
+	out = make(Params)
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	out["Alloc"] = m.Alloc
+	out["TotalAlloc"] = m.TotalAlloc
+	out["Sys"] = m.Sys
+	out["Mallocs"] = m.Mallocs
+	out["Frees"] = m.Frees
+	out["HeapAlloc"] = m.HeapAlloc
+	out["HeapSys"] = m.HeapSys
+	out["HeapIdle"] = m.HeapIdle
+	out["HeapInuse"] = m.HeapInuse
+	out["HeapReleased"] = m.HeapReleased
+	out["HeapObjects"] = m.HeapObjects
+	out["StackInuse"] = m.StackInuse
+	out["StackSys"] = m.StackSys
+	out["MSpanInuse"] = m.MSpanInuse
+	out["MSpanSys"] = m.MSpanSys
+	out["MCacheInuse"] = m.MCacheInuse
+	out["MCacheSys"] = m.MCacheSys
+	out["BuckHashSys"] = m.BuckHashSys
+	out["GCSys"] = m.GCSys
+	out["OtherSys"] = m.OtherSys
+	return out, nil
+}
+
+// Do a garbage collection run
+func rcGc(in Params) (out Params, err error) {
+	out = make(Params)
+	runtime.GC()
+	return out, nil
+}

vendor/github.com/ncw/rclone/fs/rc/rc.go

@@ -0,0 +1,146 @@
+// Package rc implements a remote control server and registry for rclone
+//
+// To register your internal calls, call rc.Add(path, function).  Your
+// function should take ane return a Param.  It can also return an
+// error.  Use rc.NewError to wrap an existing error along with an
+// http response type if another response other than 500 internal
+// error is required on error.
+package rc
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	_ "net/http/pprof" // install the pprof http handlers
+
+	"strings"
+
+	"github.com/ncw/rclone/cmd/serve/httplib"
+	"github.com/ncw/rclone/fs"
+	"github.com/pkg/errors"
+)
+
+// Options contains options for the remote control server
+type Options struct {
+	HTTPOptions httplib.Options
+	Enabled     bool
+}
+
+// DefaultOpt is the default values used for Options
+var DefaultOpt = Options{
+	HTTPOptions: httplib.DefaultOpt,
+	Enabled:     false,
+}
+
+func init() {
+	DefaultOpt.HTTPOptions.ListenAddr = "localhost:5572"
+}
+
+// Start the remote control server if configured
+func Start(opt *Options) {
+	if opt.Enabled {
+		s := newServer(opt)
+		go s.serve()
+	}
+}
+
+// server contains everything to run the server
+type server struct {
+	srv *httplib.Server
+}
+
+func newServer(opt *Options) *server {
+	// Serve on the DefaultServeMux so can have global registrations appear
+	mux := http.DefaultServeMux
+	s := &server{
+		srv: httplib.NewServer(mux, &opt.HTTPOptions),
+	}
+	mux.HandleFunc("/", s.handler)
+	return s
+}
+
+// serve runs the http server - doesn't return
+func (s *server) serve() {
+	err := s.srv.Serve()
+	if err != nil {
+		fs.Errorf(nil, "Opening listener: %v", err)
+	}
+	fs.Logf(nil, "Serving remote control on %s", s.srv.URL())
+	s.srv.Wait()
+}
+
+// WriteJSON writes JSON in out to w
+func WriteJSON(w io.Writer, out Params) error {
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "\t")
+	return enc.Encode(out)
+}
+
+// handler reads incoming requests and dispatches them
+func (s *server) handler(w http.ResponseWriter, r *http.Request) {
+	path := strings.Trim(r.URL.Path, "/")
+	in := make(Params)
+
+	writeError := func(err error, status int) {
+		fs.Errorf(nil, "rc: %q: error: %v", path, err)
+		w.WriteHeader(status)
+		err = WriteJSON(w, Params{
+			"error": err.Error(),
+			"input": in,
+		})
+		if err != nil {
+			// can't return the error at this point
+			fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
+		}
+	}
+
+	if r.Method != "POST" {
+		writeError(errors.Errorf("method %q not allowed - POST required", r.Method), http.StatusMethodNotAllowed)
+		return
+	}
+
+	// Find the call
+	call := registry.get(path)
+	if call == nil {
+		writeError(errors.Errorf("couldn't find method %q", path), http.StatusMethodNotAllowed)
+		return
+	}
+
+	// Parse the POST and URL parameters into r.Form
+	err := r.ParseForm()
+	if err != nil {
+		writeError(errors.Wrap(err, "failed to parse form/URL parameters"), http.StatusBadRequest)
+		return
+	}
+
+	// Read the POST and URL parameters into in
+	for k, vs := range r.Form {
+		if len(vs) > 0 {
+			in[k] = vs[len(vs)-1]
+		}
+	}
+	fs.Debugf(nil, "form = %+v", r.Form)
+
+	// Parse a JSON blob from the input
+	if r.Header.Get("Content-Type") == "application/json" {
+		err := json.NewDecoder(r.Body).Decode(&in)
+		if err != nil {
+			writeError(errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest)
+			return
+		}
+	}
+
+	fs.Debugf(nil, "rc: %q: with parameters %+v", path, in)
+	out, err := call.Fn(in)
+	if err != nil {
+		writeError(errors.Wrap(err, "remote control command failed"), http.StatusInternalServerError)
+		return
+	}
+
+	fs.Debugf(nil, "rc: %q: reply %+v: %v", path, out, err)
+	err = WriteJSON(w, out)
+	if err != nil {
+		// can't return the error at this point
+		fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
+	}
+}

vendor/github.com/ncw/rclone/fs/rc/registry.go

@@ -0,0 +1,79 @@
+// Define the registry
+
+package rc
+
+import (
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/ncw/rclone/fs"
+)
+
+// Params is the input and output type for the Func
+type Params map[string]interface{}
+
+// Func defines a type for a remote control function
+type Func func(in Params) (out Params, err error)
+
+// Call defines info about a remote control function and is used in
+// the Add function to create new entry points.
+type Call struct {
+	Path  string // path to activate this RC
+	Fn    Func   `json:"-"` // function to call
+	Title string // help for the function
+	Help  string // multi-line markdown formatted help
+}
+
+// Registry holds the list of all the registered remote control functions
+type Registry struct {
+	mu   sync.RWMutex
+	call map[string]*Call
+}
+
+// NewRegistry makes a new registry for remote control functions
+func NewRegistry() *Registry {
+	return &Registry{
+		call: make(map[string]*Call),
+	}
+}
+
+// Add a call to the registry
+func (r *Registry) add(call Call) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	call.Path = strings.Trim(call.Path, "/")
+	call.Help = strings.TrimSpace(call.Help)
+	fs.Debugf(nil, "Adding path %q to remote control registry", call.Path)
+	r.call[call.Path] = &call
+}
+
+// get a Call from a path or nil
+func (r *Registry) get(path string) *Call {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.call[path]
+}
+
+// get a list of all calls in alphabetical order
+func (r *Registry) list() (out []*Call) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	var keys []string
+	for key := range r.call {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		out = append(out, r.call[key])
+	}
+	return out
+}
+
+// The global registry
+var registry = NewRegistry()
+
+// Add a function to the global registry
+func Add(call Call) {
+	registry.add(call)
+}