overdue

.rclone_repo/cmd/serve/restic/restic-test.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Test all the remotes against restic integration test
+# Run with: screen -S restic-test -L ./restic-test.sh
+
+remotes="
+TestAzureBlob:
+TestB2:
+TestBox:
+TestCache:
+TestCryptDrive:
+TestCryptSwift:
+TestDrive:
+TestDropbox:
+TestFTP:
+TestGoogleCloudStorage:
+TestHubic:
+TestOneDrive:
+TestPcloud:
+TestQingStor:
+TestS3:
+TestSftp:
+TestSwift:
+TestWebdav:
+TestYandex:
+"
+
+# TestOss:
+# TestMega:
+
+for remote in $remotes; do
+    echo `date -Is` $remote starting
+    go test -remote $remote -v -timeout 30m 2>&1 | tee restic-test.$remote.log
+    echo `date -Is` $remote ending
+done

.rclone_repo/cmd/serve/restic/restic.go

@@ -0,0 +1,471 @@
+// Package restic serves a remote suitable for use with restic
+package restic
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"time"
+
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/cmd/serve/httplib"
+	"github.com/ncw/rclone/cmd/serve/httplib/httpflags"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/accounting"
+	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/ncw/rclone/fs/operations"
+	"github.com/ncw/rclone/fs/walk"
+	"github.com/spf13/cobra"
+	"golang.org/x/crypto/ssh/terminal"
+	"golang.org/x/net/http2"
+)
+
+var (
+	stdio      bool
+	appendOnly bool
+)
+
+func init() {
+	httpflags.AddFlags(Command.Flags())
+	Command.Flags().BoolVar(&stdio, "stdio", false, "run an HTTP2 server on stdin/stdout")
+	Command.Flags().BoolVar(&appendOnly, "append-only", false, "disallow deletion of repository data")
+}
+
+// Command definition for cobra
+var Command = &cobra.Command{
+	Use:   "restic remote:path",
+	Short: `Serve the remote for restic's REST API.`,
+	Long: `rclone serve restic implements restic's REST backend API
+over HTTP.  This allows restic to use rclone as a data storage
+mechanism for cloud providers that restic does not support directly.
+
+[Restic](https://restic.net/) is a command line program for doing
+backups.
+
+The server will log errors.  Use -v to see access logs.
+
+--bwlimit will be respected for file transfers.  Use --stats to
+control the stats printing.
+
+### Setting up rclone for use by restic ###
+
+First [set up a remote for your chosen cloud provider](/docs/#configure).
+
+Once you have set up the remote, check it is working with, for example
+"rclone lsd remote:".  You may have called the remote something other
+than "remote:" - just substitute whatever you called it in the
+following instructions.
+
+Now start the rclone restic server
+
+    rclone serve restic -v remote:backup
+
+Where you can replace "backup" in the above by whatever path in the
+remote you wish to use.
+
+By default this will serve on "localhost:8080" you can change this
+with use of the "--addr" flag.
+
+You might wish to start this server on boot.
+
+### Setting up restic to use rclone ###
+
+Now you can [follow the restic
+instructions](http://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server)
+on setting up restic.
+
+Note that you will need restic 0.8.2 or later to interoperate with
+rclone.
+
+For the example above you will want to use "http://localhost:8080/" as
+the URL for the REST server.
+
+For example:
+
+    $ export RESTIC_REPOSITORY=rest:http://localhost:8080/
+    $ export RESTIC_PASSWORD=yourpassword
+    $ restic init
+    created restic backend 8b1a4b56ae at rest:http://localhost:8080/
+    
+    Please note that knowledge of your password is required to access
+    the repository. Losing your password means that your data is
+    irrecoverably lost.
+    $ restic backup /path/to/files/to/backup
+    scan [/path/to/files/to/backup]
+    scanned 189 directories, 312 files in 0:00
+    [0:00] 100.00%  38.128 MiB / 38.128 MiB  501 / 501 items  0 errors  ETA 0:00 
+    duration: 0:00
+    snapshot 45c8fdd8 saved
+
+#### Multiple repositories ####
+
+Note that you can use the endpoint to host multiple repositories.  Do
+this by adding a directory name or path after the URL.  Note that
+these **must** end with /.  Eg
+
+    $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user1repo/
+    # backup user1 stuff
+    $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user2repo/
+    # backup user2 stuff
+
+` + httplib.Help,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1, command, args)
+		f := cmd.NewFsSrc(args)
+		cmd.Run(false, true, command, func() error {
+			s := newServer(f, &httpflags.Opt)
+			if stdio {
+				if terminal.IsTerminal(int(os.Stdout.Fd())) {
+					return errors.New("Refusing to run HTTP2 server directly on a terminal, please let restic start rclone")
+				}
+
+				conn := &StdioConn{
+					stdin:  os.Stdin,
+					stdout: os.Stdout,
+				}
+
+				httpSrv := &http2.Server{}
+				opts := &http2.ServeConnOpts{
+					Handler: http.HandlerFunc(s.handler),
+				}
+				httpSrv.ServeConn(conn, opts)
+				return nil
+			}
+
+			s.serve()
+			return nil
+		})
+	},
+}
+
+const (
+	resticAPIV2 = "application/vnd.x.restic.rest.v2"
+)
+
+// server contains everything to run the server
+type server struct {
+	f   fs.Fs
+	srv *httplib.Server
+}
+
+func newServer(f fs.Fs, opt *httplib.Options) *server {
+	mux := http.NewServeMux()
+	s := &server{
+		f:   f,
+		srv: httplib.NewServer(mux, opt),
+	}
+	mux.HandleFunc("/", s.handler)
+	return s
+}
+
+// serve runs the http server - doesn't return
+func (s *server) serve() {
+	err := s.srv.Serve()
+	if err != nil {
+		fs.Errorf(s.f, "Opening listener: %v", err)
+	}
+	fs.Logf(s.f, "Serving restic REST API on %s", s.srv.URL())
+	s.srv.Wait()
+}
+
+var matchData = regexp.MustCompile("(?:^|/)data/([^/]{2,})$")
+
+// Makes a remote from a URL path.  This implements the backend layout
+// required by restic.
+func makeRemote(path string) string {
+	path = strings.Trim(path, "/")
+	parts := matchData.FindStringSubmatch(path)
+	// if no data directory, layout is flat
+	if parts == nil {
+		return path
+	}
+	// otherwise map
+	// data/2159dd48 to
+	// data/21/2159dd48
+	fileName := parts[1]
+	prefix := path[:len(path)-len(fileName)]
+	return prefix + fileName[:2] + "/" + fileName
+}
+
+// handler reads incoming requests and dispatches them
+func (s *server) handler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Accept-Ranges", "bytes")
+	w.Header().Set("Server", "rclone/"+fs.Version)
+
+	path := r.URL.Path
+	remote := makeRemote(path)
+	fs.Debugf(s.f, "%s %s", r.Method, path)
+
+	// Dispatch on path then method
+	if strings.HasSuffix(path, "/") {
+		switch r.Method {
+		case "GET":
+			s.listObjects(w, r, remote)
+		case "POST":
+			s.createRepo(w, r, remote)
+		default:
+			http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
+		}
+	} else {
+		switch r.Method {
+		case "GET":
+			s.getObject(w, r, remote)
+		case "HEAD":
+			s.headObject(w, r, remote)
+		case "POST":
+			s.postObject(w, r, remote)
+		case "DELETE":
+			s.deleteObject(w, r, remote)
+		default:
+			http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
+		}
+	}
+}
+
+// head request the remote
+func (s *server) headObject(w http.ResponseWriter, r *http.Request, remote string) {
+	o, err := s.f.NewObject(remote)
+	if err != nil {
+		fs.Debugf(remote, "Head request error: %v", err)
+		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+		return
+	}
+
+	// Set content length since we know how long the object is
+	w.Header().Set("Content-Length", strconv.FormatInt(o.Size(), 10))
+}
+
+// get the remote
+func (s *server) getObject(w http.ResponseWriter, r *http.Request, remote string) {
+	o, err := s.f.NewObject(remote)
+	if err != nil {
+		fs.Debugf(remote, "Get request error: %v", err)
+		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+		return
+	}
+
+	// Set content length since we know how long the object is
+	w.Header().Set("Content-Length", strconv.FormatInt(o.Size(), 10))
+
+	// Decode Range request if present
+	code := http.StatusOK
+	size := o.Size()
+	var options []fs.OpenOption
+	if rangeRequest := r.Header.Get("Range"); rangeRequest != "" {
+		//fs.Debugf(nil, "Range: request %q", rangeRequest)
+		option, err := fs.ParseRangeOption(rangeRequest)
+		if err != nil {
+			fs.Debugf(remote, "Get request parse range request error: %v", err)
+			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+			return
+		}
+		options = append(options, option)
+		offset, limit := option.Decode(o.Size())
+		end := o.Size() // exclusive
+		if limit >= 0 {
+			end = offset + limit
+		}
+		if end > o.Size() {
+			end = o.Size()
+		}
+		size = end - offset
+		// fs.Debugf(nil, "Range: offset=%d, limit=%d, end=%d, size=%d (object size %d)", offset, limit, end, size, o.Size())
+		// Content-Range: bytes 0-1023/146515
+		w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, end-1, o.Size()))
+		// fs.Debugf(nil, "Range: Content-Range: %q", w.Header().Get("Content-Range"))
+		code = http.StatusPartialContent
+	}
+	w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
+
+	file, err := o.Open(options...)
+	if err != nil {
+		fs.Debugf(remote, "Get request open error: %v", err)
+		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+		return
+	}
+	accounting.Stats.Transferring(o.Remote())
+	in := accounting.NewAccount(file, o) // account the transfer (no buffering)
+	defer func() {
+		closeErr := in.Close()
+		if closeErr != nil {
+			fs.Errorf(remote, "Get request: close failed: %v", closeErr)
+			if err == nil {
+				err = closeErr
+			}
+		}
+		ok := err == nil
+		accounting.Stats.DoneTransferring(o.Remote(), ok)
+		if !ok {
+			accounting.Stats.Error(err)
+		}
+	}()
+
+	w.WriteHeader(code)
+
+	n, err := io.Copy(w, in)
+	if err != nil {
+		fs.Errorf(remote, "Didn't finish writing GET request (wrote %d/%d bytes): %v", n, size, err)
+		return
+	}
+}
+
+// postObject posts an object to the repository
+func (s *server) postObject(w http.ResponseWriter, r *http.Request, remote string) {
+	if appendOnly {
+		// make sure the file does not exist yet
+		_, err := s.f.NewObject(remote)
+		if err == nil {
+			fs.Errorf(remote, "Post request: file already exists, refusing to overwrite in append-only mode")
+			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
+
+			return
+		}
+	}
+
+	_, err := operations.RcatSize(s.f, remote, r.Body, r.ContentLength, time.Now())
+	if err != nil {
+		accounting.Stats.Error(err)
+		fs.Errorf(remote, "Post request rcat error: %v", err)
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+
+		return
+	}
+}
+
+// delete the remote
+func (s *server) deleteObject(w http.ResponseWriter, r *http.Request, remote string) {
+	if appendOnly {
+		parts := strings.Split(r.URL.Path, "/")
+
+		// if path doesn't end in "/locks/:name", disallow the operation
+		if len(parts) < 2 || parts[len(parts)-2] != "locks" {
+			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
+			return
+		}
+	}
+
+	o, err := s.f.NewObject(remote)
+	if err != nil {
+		fs.Debugf(remote, "Delete request error: %v", err)
+		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+		return
+	}
+
+	if err := o.Remove(); err != nil {
+		fs.Errorf(remote, "Delete request remove error: %v", err)
+		if err == fs.ErrorObjectNotFound {
+			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+		} else {
+			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		}
+		return
+	}
+}
+
+// listItem is an element returned for the restic v2 list response
+type listItem struct {
+	Name string `json:"name"`
+	Size int64  `json:"size"`
+}
+
+// return type for list
+type listItems []listItem
+
+// add a DirEntry to the listItems
+func (ls *listItems) add(entry fs.DirEntry) {
+	if o, ok := entry.(fs.Object); ok {
+		*ls = append(*ls, listItem{
+			Name: path.Base(o.Remote()),
+			Size: o.Size(),
+		})
+	}
+}
+
+// listObjects lists all Objects of a given type in an arbitrary order.
+func (s *server) listObjects(w http.ResponseWriter, r *http.Request, remote string) {
+	fs.Debugf(remote, "list request")
+
+	if r.Header.Get("Accept") != resticAPIV2 {
+		fs.Errorf(remote, "Restic v2 API required")
+		http.Error(w, "Restic v2 API required", http.StatusBadRequest)
+		return
+	}
+
+	// make sure an empty list is returned, and not a 'nil' value
+	ls := listItems{}
+
+	// if remote supports ListR use that directly, otherwise use recursive Walk
+	var err error
+	if ListR := s.f.Features().ListR; ListR != nil {
+		err = ListR(remote, func(entries fs.DirEntries) error {
+			for _, entry := range entries {
+				ls.add(entry)
+			}
+			return nil
+		})
+	} else {
+		err = walk.Walk(s.f, remote, true, -1, func(path string, entries fs.DirEntries, err error) error {
+			if err == nil {
+				for _, entry := range entries {
+					ls.add(entry)
+				}
+			}
+			return err
+		})
+	}
+
+	if err != nil {
+		_, err = fserrors.Cause(err)
+		if err != fs.ErrorDirNotFound {
+			fs.Errorf(remote, "list failed: %#v %T", err, err)
+			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+			return
+		}
+	}
+
+	w.Header().Set("Content-Type", "application/vnd.x.restic.rest.v2")
+	enc := json.NewEncoder(w)
+	err = enc.Encode(ls)
+	if err != nil {
+		fs.Errorf(remote, "failed to write list: %v", err)
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
+}
+
+// createRepo creates repository directories.
+//
+// We don't bother creating the data dirs as rclone will create them on the fly
+func (s *server) createRepo(w http.ResponseWriter, r *http.Request, remote string) {
+	fs.Infof(remote, "Creating repository")
+
+	if r.URL.Query().Get("create") != "true" {
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+
+	err := s.f.Mkdir(remote)
+	if err != nil {
+		fs.Errorf(remote, "Create repo failed to Mkdir: %v", err)
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
+
+	for _, name := range []string{"data", "index", "keys", "locks", "snapshots"} {
+		dirRemote := path.Join(remote, name)
+		err := s.f.Mkdir(dirRemote)
+		if err != nil {
+			fs.Errorf(dirRemote, "Create repo failed to Mkdir: %v", err)
+			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+			return
+		}
+	}
+}

.rclone_repo/cmd/serve/restic/restic_appendonly_test.go

@@ -0,0 +1,190 @@
+package restic
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/cmd/serve/httplib/httpflags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// declare a few helper functions
+
+// wantFunc tests the HTTP response in res and marks the test as errored if something is incorrect.
+type wantFunc func(t testing.TB, res *httptest.ResponseRecorder)
+
+// newRequest returns a new HTTP request with the given params. On error, the
+// test is marked as failed.
+func newRequest(t testing.TB, method, path string, body io.Reader) *http.Request {
+	req, err := http.NewRequest(method, path, body)
+	require.NoError(t, err)
+	return req
+}
+
+// wantCode returns a function which checks that the response has the correct HTTP status code.
+func wantCode(code int) wantFunc {
+	return func(t testing.TB, res *httptest.ResponseRecorder) {
+		assert.Equal(t, code, res.Code)
+	}
+}
+
+// wantBody returns a function which checks that the response has the data in the body.
+func wantBody(body string) wantFunc {
+	return func(t testing.TB, res *httptest.ResponseRecorder) {
+		assert.NotNil(t, res.Body)
+		assert.Equal(t, res.Body.Bytes(), []byte(body))
+	}
+}
+
+// checkRequest uses f to process the request and runs the checker functions on the result.
+func checkRequest(t testing.TB, f http.HandlerFunc, req *http.Request, want []wantFunc) {
+	rr := httptest.NewRecorder()
+	f(rr, req)
+
+	for _, fn := range want {
+		fn(t, rr)
+	}
+}
+
+// TestRequest is a sequence of HTTP requests with (optional) tests for the response.
+type TestRequest struct {
+	req  *http.Request
+	want []wantFunc
+}
+
+// createOverwriteDeleteSeq returns a sequence which will create a new file at
+// path, and then try to overwrite and delete it.
+func createOverwriteDeleteSeq(t testing.TB, path string) []TestRequest {
+	// add a file, try to overwrite and delete it
+	req := []TestRequest{
+		{
+			req:  newRequest(t, "GET", path, nil),
+			want: []wantFunc{wantCode(http.StatusNotFound)},
+		},
+		{
+			req:  newRequest(t, "POST", path, strings.NewReader("foobar test config")),
+			want: []wantFunc{wantCode(http.StatusOK)},
+		},
+		{
+			req: newRequest(t, "GET", path, nil),
+			want: []wantFunc{
+				wantCode(http.StatusOK),
+				wantBody("foobar test config"),
+			},
+		},
+		{
+			req:  newRequest(t, "POST", path, strings.NewReader("other config")),
+			want: []wantFunc{wantCode(http.StatusForbidden)},
+		},
+		{
+			req: newRequest(t, "GET", path, nil),
+			want: []wantFunc{
+				wantCode(http.StatusOK),
+				wantBody("foobar test config"),
+			},
+		},
+		{
+			req:  newRequest(t, "DELETE", path, nil),
+			want: []wantFunc{wantCode(http.StatusForbidden)},
+		},
+		{
+			req: newRequest(t, "GET", path, nil),
+			want: []wantFunc{
+				wantCode(http.StatusOK),
+				wantBody("foobar test config"),
+			},
+		},
+	}
+	return req
+}
+
+// TestResticHandler runs tests on the restic handler code, especially in append-only mode.
+func TestResticHandler(t *testing.T) {
+	buf := make([]byte, 32)
+	_, err := io.ReadFull(rand.Reader, buf)
+	require.NoError(t, err)
+	randomID := hex.EncodeToString(buf)
+
+	var tests = []struct {
+		seq []TestRequest
+	}{
+		{createOverwriteDeleteSeq(t, "/config")},
+		{createOverwriteDeleteSeq(t, "/data/"+randomID)},
+		{
+			// ensure we can add and remove lock files
+			[]TestRequest{
+				{
+					req:  newRequest(t, "GET", "/locks/"+randomID, nil),
+					want: []wantFunc{wantCode(http.StatusNotFound)},
+				},
+				{
+					req:  newRequest(t, "POST", "/locks/"+randomID, strings.NewReader("lock file")),
+					want: []wantFunc{wantCode(http.StatusOK)},
+				},
+				{
+					req: newRequest(t, "GET", "/locks/"+randomID, nil),
+					want: []wantFunc{
+						wantCode(http.StatusOK),
+						wantBody("lock file"),
+					},
+				},
+				{
+					req:  newRequest(t, "POST", "/locks/"+randomID, strings.NewReader("other lock file")),
+					want: []wantFunc{wantCode(http.StatusForbidden)},
+				},
+				{
+					req:  newRequest(t, "DELETE", "/locks/"+randomID, nil),
+					want: []wantFunc{wantCode(http.StatusOK)},
+				},
+				{
+					req:  newRequest(t, "GET", "/locks/"+randomID, nil),
+					want: []wantFunc{wantCode(http.StatusNotFound)},
+				},
+			},
+		},
+	}
+
+	// setup rclone with a local backend in a temporary directory
+	tempdir, err := ioutil.TempDir("", "rclone-restic-test-")
+	require.NoError(t, err)
+
+	// make sure the tempdir is properly removed
+	defer func() {
+		err := os.RemoveAll(tempdir)
+		require.NoError(t, err)
+	}()
+
+	// globally set append-only mode
+	prev := appendOnly
+	appendOnly = true
+	defer func() {
+		appendOnly = prev // reset when done
+	}()
+
+	// make a new file system in the temp dir
+	f := cmd.NewFsSrc([]string{tempdir})
+	srv := newServer(f, &httpflags.Opt)
+
+	// create the repo
+	checkRequest(t, srv.handler,
+		newRequest(t, "POST", "/?create=true", nil),
+		[]wantFunc{wantCode(http.StatusOK)})
+
+	for _, test := range tests {
+		t.Run("", func(t *testing.T) {
+			for i, seq := range test.seq {
+				t.Logf("request %v: %v %v", i, seq.req.Method, seq.req.URL.Path)
+				checkRequest(t, srv.handler, seq.req, seq.want)
+			}
+		})
+	}
+}

.rclone_repo/cmd/serve/restic/restic_test.go

@@ -0,0 +1,95 @@
+// Serve restic tests set up a server and run the integration tests
+// for restic against it.
+package restic
+
+import (
+	"os"
+	"os/exec"
+	"testing"
+
+	_ "github.com/ncw/rclone/backend/all"
+	"github.com/ncw/rclone/cmd/serve/httplib"
+	"github.com/ncw/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testBindAddress = "localhost:51779"
+	testURL         = "http://" + testBindAddress + "/"
+	resticSource    = "../../../../../restic/restic"
+)
+
+// TestRestic runs the restic server then runs the unit tests for the
+// restic remote against it.
+func TestRestic(t *testing.T) {
+	_, err := os.Stat(resticSource)
+	if err != nil {
+		t.Skipf("Skipping test as restic source not found: %v", err)
+	}
+
+	opt := httplib.DefaultOpt
+	opt.ListenAddr = testBindAddress
+
+	fstest.Initialise()
+
+	fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
+	assert.NoError(t, err)
+	defer clean()
+
+	err = fremote.Mkdir("")
+	assert.NoError(t, err)
+
+	// Start the server
+	w := newServer(fremote, &opt)
+	go w.serve()
+	defer w.srv.Close()
+
+	// Change directory to run the tests
+	err = os.Chdir(resticSource)
+	assert.NoError(t, err, "failed to cd to restic source code")
+
+	// Run the restic tests
+	runTests := func(path string) {
+		args := []string{"test", "./internal/backend/rest", "-run", "TestBackendRESTExternalServer", "-count=1"}
+		if testing.Verbose() {
+			args = append(args, "-v")
+		}
+		cmd := exec.Command("go", args...)
+		cmd.Env = append(os.Environ(),
+			"RESTIC_TEST_REST_REPOSITORY=rest:"+testURL+path,
+		)
+		out, err := cmd.CombinedOutput()
+		if len(out) != 0 {
+			t.Logf("\n----------\n%s----------\n", string(out))
+		}
+		assert.NoError(t, err, "Running restic integration tests")
+	}
+
+	// Run the tests with no path
+	runTests("")
+	//... and again with a path
+	runTests("potato/sausage/")
+
+}
+
+func TestMakeRemote(t *testing.T) {
+	for _, test := range []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"/", ""},
+		{"/data", "data"},
+		{"/data/", "data"},
+		{"/data/1", "data/1"},
+		{"/data/12", "data/12/12"},
+		{"/data/123", "data/12/123"},
+		{"/data/123/", "data/12/123"},
+		{"/keys", "keys"},
+		{"/keys/1", "keys/1"},
+		{"/keys/12", "keys/12"},
+		{"/keys/123", "keys/123"},
+	} {
+		got := makeRemote(test.in)
+		assert.Equal(t, test.want, got, test.in)
+	}
+}

.rclone_repo/cmd/serve/restic/stdio_conn.go

@@ -0,0 +1,52 @@
+package restic
+
+import (
+	"net"
+	"os"
+)
+
+// Addr implements net.Addr for stdin/stdout.
+type Addr struct{}
+
+// Network returns the network type as a string.
+func (a Addr) Network() string {
+	return "stdio"
+}
+
+func (a Addr) String() string {
+	return "stdio"
+}
+
+// StdioConn implements a net.Conn via stdin/stdout.
+type StdioConn struct {
+	stdin  *os.File
+	stdout *os.File
+}
+
+func (s *StdioConn) Read(p []byte) (int, error) {
+	return s.stdin.Read(p)
+}
+
+func (s *StdioConn) Write(p []byte) (int, error) {
+	return s.stdout.Write(p)
+}
+
+// Close closes both streams.
+func (s *StdioConn) Close() error {
+	err1 := s.stdin.Close()
+	err2 := s.stdout.Close()
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+// LocalAddr returns nil.
+func (s *StdioConn) LocalAddr() net.Addr {
+	return Addr{}
+}
+
+// RemoteAddr returns nil.
+func (s *StdioConn) RemoteAddr() net.Addr {
+	return Addr{}
+}

.rclone_repo/cmd/serve/restic/stdio_conn_go1.10.go

@@ -0,0 +1,27 @@
+//+build go1.10
+
+// Deadline setting for go1.10+
+
+package restic
+
+import "time"
+
+// SetDeadline sets the read/write deadline.
+func (s *StdioConn) SetDeadline(t time.Time) error {
+	err1 := s.stdin.SetReadDeadline(t)
+	err2 := s.stdout.SetWriteDeadline(t)
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+// SetReadDeadline sets the read/write deadline.
+func (s *StdioConn) SetReadDeadline(t time.Time) error {
+	return s.stdin.SetReadDeadline(t)
+}
+
+// SetWriteDeadline sets the read/write deadline.
+func (s *StdioConn) SetWriteDeadline(t time.Time) error {
+	return s.stdout.SetWriteDeadline(t)
+}

.rclone_repo/cmd/serve/restic/stdio_conn_pre_go1.10.go

@@ -0,0 +1,22 @@
+//+build !go1.10
+
+// Fallback deadline setting for pre go1.10
+
+package restic
+
+import "time"
+
+// SetDeadline sets the read/write deadline.
+func (s *StdioConn) SetDeadline(t time.Time) error {
+	return nil
+}
+
+// SetReadDeadline sets the read/write deadline.
+func (s *StdioConn) SetReadDeadline(t time.Time) error {
+	return nil
+}
+
+// SetWriteDeadline sets the read/write deadline.
+func (s *StdioConn) SetWriteDeadline(t time.Time) error {
+	return nil
+}