overdue

.rclone_repo/fs/fserrors/error.go

@@ -0,0 +1,292 @@
+// Package fserrors provides errors and error handling
+package fserrors
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"reflect"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// Retrier is an optional interface for error as to whether the
+// operation should be retried at a high level.
+//
+// This should be returned from Update or Put methods as required
+type Retrier interface {
+	error
+	Retry() bool
+}
+
+// retryError is a type of error
+type retryError string
+
+// Error interface
+func (r retryError) Error() string {
+	return string(r)
+}
+
+// Retry interface
+func (r retryError) Retry() bool {
+	return true
+}
+
+// Check interface
+var _ Retrier = retryError("")
+
+// RetryErrorf makes an error which indicates it would like to be retried
+func RetryErrorf(format string, a ...interface{}) error {
+	return retryError(fmt.Sprintf(format, a...))
+}
+
+// wrappedRetryError is an error wrapped so it will satisfy the
+// Retrier interface and return true
+type wrappedRetryError struct {
+	error
+}
+
+// Retry interface
+func (err wrappedRetryError) Retry() bool {
+	return true
+}
+
+// Check interface
+var _ Retrier = wrappedRetryError{error(nil)}
+
+// RetryError makes an error which indicates it would like to be retried
+func RetryError(err error) error {
+	if err == nil {
+		err = errors.New("needs retry")
+	}
+	return wrappedRetryError{err}
+}
+
+// IsRetryError returns true if err conforms to the Retry interface
+// and calling the Retry method returns true.
+func IsRetryError(err error) bool {
+	if err == nil {
+		return false
+	}
+	_, err = Cause(err)
+	if r, ok := err.(Retrier); ok {
+		return r.Retry()
+	}
+	return false
+}
+
+// Fataler is an optional interface for error as to whether the
+// operation should cause the entire operation to finish immediately.
+//
+// This should be returned from Update or Put methods as required
+type Fataler interface {
+	error
+	Fatal() bool
+}
+
+// wrappedFatalError is an error wrapped so it will satisfy the
+// Retrier interface and return true
+type wrappedFatalError struct {
+	error
+}
+
+// Fatal interface
+func (err wrappedFatalError) Fatal() bool {
+	return true
+}
+
+// Check interface
+var _ Fataler = wrappedFatalError{error(nil)}
+
+// FatalError makes an error which indicates it is a fatal error and
+// the sync should stop.
+func FatalError(err error) error {
+	if err == nil {
+		err = errors.New("fatal error")
+	}
+	return wrappedFatalError{err}
+}
+
+// IsFatalError returns true if err conforms to the Fatal interface
+// and calling the Fatal method returns true.
+func IsFatalError(err error) bool {
+	if err == nil {
+		return false
+	}
+	_, err = Cause(err)
+	if r, ok := err.(Fataler); ok {
+		return r.Fatal()
+	}
+	return false
+}
+
+// NoRetrier is an optional interface for error as to whether the
+// operation should not be retried at a high level.
+//
+// If only NoRetry errors are returned in a sync then the sync won't
+// be retried.
+//
+// This should be returned from Update or Put methods as required
+type NoRetrier interface {
+	error
+	NoRetry() bool
+}
+
+// wrappedNoRetryError is an error wrapped so it will satisfy the
+// Retrier interface and return true
+type wrappedNoRetryError struct {
+	error
+}
+
+// NoRetry interface
+func (err wrappedNoRetryError) NoRetry() bool {
+	return true
+}
+
+// Check interface
+var _ NoRetrier = wrappedNoRetryError{error(nil)}
+
+// NoRetryError makes an error which indicates the sync shouldn't be
+// retried.
+func NoRetryError(err error) error {
+	return wrappedNoRetryError{err}
+}
+
+// IsNoRetryError returns true if err conforms to the NoRetry
+// interface and calling the NoRetry method returns true.
+func IsNoRetryError(err error) bool {
+	if err == nil {
+		return false
+	}
+	_, err = Cause(err)
+	if r, ok := err.(NoRetrier); ok {
+		return r.NoRetry()
+	}
+	return false
+}
+
+// Cause is a souped up errors.Cause which can unwrap some standard
+// library errors too.  It returns true if any of the intermediate
+// errors had a Timeout() or Temporary() method which returned true.
+func Cause(cause error) (retriable bool, err error) {
+	err = cause
+	for prev := err; err != nil; prev = err {
+		// Check for net error Timeout()
+		if x, ok := err.(interface {
+			Timeout() bool
+		}); ok && x.Timeout() {
+			retriable = true
+		}
+
+		// Check for net error Temporary()
+		if x, ok := err.(interface {
+			Temporary() bool
+		}); ok && x.Temporary() {
+			retriable = true
+		}
+
+		// Unwrap 1 level if possible
+		err = errors.Cause(err)
+		if err == nil {
+			// errors.Cause can return nil which isn't
+			// desirable so pick the previous error in
+			// this case.
+			err = prev
+		}
+		if err == prev {
+			// Unpack any struct or *struct with a field
+			// of name Err which satisfies the error
+			// interface.  This includes *url.Error,
+			// *net.OpError, *os.SyscallError and many
+			// others in the stdlib
+			errType := reflect.TypeOf(err)
+			errValue := reflect.ValueOf(err)
+			if errValue.IsValid() && errType.Kind() == reflect.Ptr {
+				errType = errType.Elem()
+				errValue = errValue.Elem()
+			}
+			if errValue.IsValid() && errType.Kind() == reflect.Struct {
+				if errField := errValue.FieldByName("Err"); errField.IsValid() {
+					errFieldValue := errField.Interface()
+					if newErr, ok := errFieldValue.(error); ok {
+						err = newErr
+					}
+				}
+			}
+		}
+		if err == prev {
+			break
+		}
+	}
+	return retriable, err
+}
+
+// retriableErrorStrings is a list of phrases which when we find it
+// in an an error, we know it is a networking error which should be
+// retried.
+//
+// This is incredibly ugly - if only errors.Cause worked for all
+// errors and all errors were exported from the stdlib.
+var retriableErrorStrings = []string{
+	"use of closed network connection", // internal/poll/fd.go
+	"unexpected EOF reading trailer",   // net/http/transfer.go
+	"transport connection broken",      // net/http/transport.go
+	"http: ContentLength=",             // net/http/transfer.go
+}
+
+// Errors which indicate networking errors which should be retried
+//
+// These are added to in retriable_errors*.go
+var retriableErrors = []error{
+	io.EOF,
+	io.ErrUnexpectedEOF,
+}
+
+// ShouldRetry looks at an error and tries to work out if retrying the
+// operation that caused it would be a good idea. It returns true if
+// the error implements Timeout() or Temporary() or if the error
+// indicates a premature closing of the connection.
+func ShouldRetry(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Find root cause if available
+	retriable, err := Cause(err)
+	if retriable {
+		return true
+	}
+
+	// Check if it is a retriable error
+	for _, retriableErr := range retriableErrors {
+		if err == retriableErr {
+			return true
+		}
+	}
+
+	// Check error strings (yuch!) too
+	errString := err.Error()
+	for _, phrase := range retriableErrorStrings {
+		if strings.Contains(errString, phrase) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// ShouldRetryHTTP returns a boolean as to whether this resp deserves.
+// It checks to see if the HTTP response code is in the slice
+// retryErrorCodes.
+func ShouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool {
+	if resp == nil {
+		return false
+	}
+	for _, e := range retryErrorCodes {
+		if resp.StatusCode == e {
+			return true
+		}
+	}
+	return false
+}

.rclone_repo/fs/fserrors/error_test.go

@@ -0,0 +1,145 @@
+package fserrors
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/url"
+	"os"
+	"syscall"
+	"testing"
+
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+var errUseOfClosedNetworkConnection = errors.New("use of closed network connection")
+
+// make a plausible network error with the underlying errno
+func makeNetErr(errno syscall.Errno) error {
+	return &net.OpError{
+		Op:     "write",
+		Net:    "tcp",
+		Source: &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 123},
+		Addr:   &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8080},
+		Err: &os.SyscallError{
+			Syscall: "write",
+			Err:     errno,
+		},
+	}
+}
+
+type myError1 struct {
+	Err error
+}
+
+func (e myError1) Error() string { return e.Err.Error() }
+
+type myError2 struct {
+	Err error
+}
+
+func (e *myError2) Error() string {
+	if e == nil {
+		return "myError2(nil)"
+	}
+	if e.Err == nil {
+		return "myError2{Err: nil}"
+	}
+	return e.Err.Error()
+}
+
+type myError3 struct {
+	Err int
+}
+
+func (e *myError3) Error() string { return "hello" }
+
+type myError4 struct {
+	e error
+}
+
+func (e *myError4) Error() string { return e.e.Error() }
+
+type errorCause struct {
+	e error
+}
+
+func (e *errorCause) Error() string { return fmt.Sprintf("%#v", e) }
+
+func (e *errorCause) Cause() error { return e.e }
+
+func TestCause(t *testing.T) {
+	e3 := &myError3{3}
+	e4 := &myError4{io.EOF}
+	eNil1 := &myError2{nil}
+	eNil2 := &myError2{Err: (*myError2)(nil)}
+	errPotato := errors.New("potato")
+	nilCause1 := &errorCause{nil}
+	nilCause2 := &errorCause{(*myError2)(nil)}
+
+	for i, test := range []struct {
+		err           error
+		wantRetriable bool
+		wantErr       error
+	}{
+		{nil, false, nil},
+		{errPotato, false, errPotato},
+		{errors.Wrap(errPotato, "potato"), false, errPotato},
+		{errors.Wrap(errors.Wrap(errPotato, "potato2"), "potato"), false, errPotato},
+		{errUseOfClosedNetworkConnection, false, errUseOfClosedNetworkConnection},
+		{makeNetErr(syscall.EAGAIN), true, syscall.EAGAIN},
+		{makeNetErr(syscall.Errno(123123123)), false, syscall.Errno(123123123)},
+		{eNil1, false, eNil1},
+		{eNil2, false, eNil2.Err},
+		{myError1{io.EOF}, false, io.EOF},
+		{&myError2{io.EOF}, false, io.EOF},
+		{e3, false, e3},
+		{e4, false, e4},
+		{&errorCause{errPotato}, false, errPotato},
+		{nilCause1, false, nilCause1},
+		{nilCause2, false, nilCause2.e},
+	} {
+		gotRetriable, gotErr := Cause(test.err)
+		what := fmt.Sprintf("test #%d: %v", i, test.err)
+		assert.Equal(t, test.wantErr, gotErr, what)
+		assert.Equal(t, test.wantRetriable, gotRetriable, what)
+	}
+}
+
+func TestShouldRetry(t *testing.T) {
+	for i, test := range []struct {
+		err  error
+		want bool
+	}{
+		{nil, false},
+		{errors.New("potato"), false},
+		{errors.Wrap(errUseOfClosedNetworkConnection, "connection"), true},
+		{io.EOF, true},
+		{io.ErrUnexpectedEOF, true},
+		{makeNetErr(syscall.EAGAIN), true},
+		{makeNetErr(syscall.Errno(123123123)), false},
+		{&url.Error{Op: "post", URL: "/", Err: io.EOF}, true},
+		{&url.Error{Op: "post", URL: "/", Err: errUseOfClosedNetworkConnection}, true},
+		{&url.Error{Op: "post", URL: "/", Err: fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", fmt.Errorf("http: ContentLength=%d with Body length %d", 100663336, 99590598))}, true},
+		{
+			errors.Wrap(&url.Error{
+				Op:  "post",
+				URL: "http://localhost/",
+				Err: makeNetErr(syscall.EPIPE),
+			}, "potato error"),
+			true,
+		},
+		{
+			errors.Wrap(&url.Error{
+				Op:  "post",
+				URL: "http://localhost/",
+				Err: makeNetErr(syscall.Errno(123123123)),
+			}, "listing error"),
+			false,
+		},
+	} {
+		got := ShouldRetry(test.err)
+		assert.Equal(t, test.want, got, fmt.Sprintf("test #%d: %v", i, test.err))
+	}
+}

.rclone_repo/fs/fserrors/retriable_errors.go

@@ -0,0 +1,21 @@
+// +build !plan9
+
+package fserrors
+
+import (
+	"syscall"
+)
+
+func init() {
+	retriableErrors = append(retriableErrors,
+		syscall.EPIPE,
+		syscall.ETIMEDOUT,
+		syscall.ECONNREFUSED,
+		syscall.EHOSTDOWN,
+		syscall.EHOSTUNREACH,
+		syscall.ECONNABORTED,
+		syscall.EAGAIN,
+		syscall.EWOULDBLOCK,
+		syscall.ECONNRESET,
+	)
+}

.rclone_repo/fs/fserrors/retriable_errors_windows.go

@@ -0,0 +1,31 @@
+// +build windows
+
+package fserrors
+
+import (
+	"syscall"
+)
+
+const (
+	WSAECONNABORTED   syscall.Errno = 10053
+	WSAHOST_NOT_FOUND syscall.Errno = 11001
+	WSATRY_AGAIN      syscall.Errno = 11002
+	WSAENETRESET      syscall.Errno = 10052
+	WSAETIMEDOUT      syscall.Errno = 10060
+)
+
+func init() {
+	// append some lower level errors since the standardized ones
+	// don't seem to happen
+	retriableErrors = append(retriableErrors,
+		syscall.WSAECONNRESET,
+		WSAECONNABORTED,
+		WSAHOST_NOT_FOUND,
+		WSATRY_AGAIN,
+		WSAENETRESET,
+		WSAETIMEDOUT,
+		syscall.ERROR_HANDLE_EOF,
+		syscall.ERROR_NETNAME_DELETED,
+		syscall.ERROR_BROKEN_PIPE,
+	)
+}