newline battles continue

vendor/github.com/ncw/rclone/fs/accounting/accounting.go

@@ -0,0 +1,391 @@
+// Package accounting providers an accounting and limiting reader
+package accounting
+
+import (
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/asyncreader"
+	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/pkg/errors"
+)
+
+// ErrorMaxTransferLimitReached is returned from Read when the max
+// transfer limit is reached.
+var ErrorMaxTransferLimitReached = fserrors.FatalError(errors.New("Max transfer limit reached as set by --max-transfer"))
+
+// Account limits and accounts for one transfer
+type Account struct {
+	// The mutex is to make sure Read() and Close() aren't called
+	// concurrently.  Unfortunately the persistent connection loop
+	// in http transport calls Read() after Do() returns on
+	// CancelRequest so this race can happen when it apparently
+	// shouldn't.
+	mu      sync.Mutex
+	in      io.Reader
+	origIn  io.ReadCloser
+	close   io.Closer
+	size    int64
+	name    string
+	statmu  sync.Mutex    // Separate mutex for stat values.
+	bytes   int64         // Total number of bytes read
+	max     int64         // if >=0 the max number of bytes to transfer
+	start   time.Time     // Start time of first read
+	lpTime  time.Time     // Time of last average measurement
+	lpBytes int           // Number of bytes read since last measurement
+	avg     float64       // Moving average of last few measurements in bytes/s
+	closed  bool          // set if the file is closed
+	exit    chan struct{} // channel that will be closed when transfer is finished
+	withBuf bool          // is using a buffered in
+}
+
+const averagePeriod = 16 // period to do exponentially weighted averages over
+
+// NewAccountSizeName makes a Account reader for an io.ReadCloser of
+// the given size and name
+func NewAccountSizeName(in io.ReadCloser, size int64, name string) *Account {
+	acc := &Account{
+		in:     in,
+		close:  in,
+		origIn: in,
+		size:   size,
+		name:   name,
+		exit:   make(chan struct{}),
+		avg:    0,
+		lpTime: time.Now(),
+		max:    int64(fs.Config.MaxTransfer),
+	}
+	go acc.averageLoop()
+	Stats.inProgress.set(acc.name, acc)
+	return acc
+}
+
+// NewAccount makes a Account reader for an object
+func NewAccount(in io.ReadCloser, obj fs.Object) *Account {
+	return NewAccountSizeName(in, obj.Size(), obj.Remote())
+}
+
+// WithBuffer - If the file is above a certain size it adds an Async reader
+func (acc *Account) WithBuffer() *Account {
+	acc.withBuf = true
+	var buffers int
+	if acc.size >= int64(fs.Config.BufferSize) || acc.size == -1 {
+		buffers = int(int64(fs.Config.BufferSize) / asyncreader.BufferSize)
+	} else {
+		buffers = int(acc.size / asyncreader.BufferSize)
+	}
+	// On big files add a buffer
+	if buffers > 0 {
+		rc, err := asyncreader.New(acc.origIn, buffers)
+		if err != nil {
+			fs.Errorf(acc.name, "Failed to make buffer: %v", err)
+		} else {
+			acc.in = rc
+			acc.close = rc
+		}
+	}
+	return acc
+}
+
+// GetReader returns the underlying io.ReadCloser under any Buffer
+func (acc *Account) GetReader() io.ReadCloser {
+	acc.mu.Lock()
+	defer acc.mu.Unlock()
+	return acc.origIn
+}
+
+// GetAsyncReader returns the current AsyncReader or nil if Account is unbuffered
+func (acc *Account) GetAsyncReader() *asyncreader.AsyncReader {
+	acc.mu.Lock()
+	defer acc.mu.Unlock()
+	if asyncIn, ok := acc.in.(*asyncreader.AsyncReader); ok {
+		return asyncIn
+	}
+	return nil
+}
+
+// StopBuffering stops the async buffer doing any more buffering
+func (acc *Account) StopBuffering() {
+	if asyncIn, ok := acc.in.(*asyncreader.AsyncReader); ok {
+		asyncIn.Abandon()
+	}
+}
+
+// UpdateReader updates the underlying io.ReadCloser stopping the
+// asynb buffer (if any) and re-adding it
+func (acc *Account) UpdateReader(in io.ReadCloser) {
+	acc.mu.Lock()
+	acc.StopBuffering()
+	acc.in = in
+	acc.close = in
+	acc.origIn = in
+	acc.WithBuffer()
+	acc.mu.Unlock()
+}
+
+// averageLoop calculates averages for the stats in the background
+func (acc *Account) averageLoop() {
+	tick := time.NewTicker(time.Second)
+	var period float64
+	defer tick.Stop()
+	for {
+		select {
+		case now := <-tick.C:
+			acc.statmu.Lock()
+			// Add average of last second.
+			elapsed := now.Sub(acc.lpTime).Seconds()
+			avg := float64(acc.lpBytes) / elapsed
+			// Soft start the moving average
+			if period < averagePeriod {
+				period++
+			}
+			acc.avg = (avg + (period-1)*acc.avg) / period
+			acc.lpBytes = 0
+			acc.lpTime = now
+			// Unlock stats
+			acc.statmu.Unlock()
+		case <-acc.exit:
+			return
+		}
+	}
+}
+
+// read bytes from the io.Reader passed in and account them
+func (acc *Account) read(in io.Reader, p []byte) (n int, err error) {
+	acc.statmu.Lock()
+	if acc.max >= 0 && Stats.GetBytes() >= acc.max {
+		acc.statmu.Unlock()
+		return 0, ErrorMaxTransferLimitReached
+	}
+	// Set start time.
+	if acc.start.IsZero() {
+		acc.start = time.Now()
+	}
+	acc.statmu.Unlock()
+
+	n, err = in.Read(p)
+
+	// Update Stats
+	acc.statmu.Lock()
+	acc.lpBytes += n
+	acc.bytes += int64(n)
+	acc.statmu.Unlock()
+
+	Stats.Bytes(int64(n))
+
+	limitBandwidth(n)
+	return
+}
+
+// Read bytes from the object - see io.Reader
+func (acc *Account) Read(p []byte) (n int, err error) {
+	acc.mu.Lock()
+	defer acc.mu.Unlock()
+	return acc.read(acc.in, p)
+}
+
+// Close the object
+func (acc *Account) Close() error {
+	acc.mu.Lock()
+	defer acc.mu.Unlock()
+	if acc.closed {
+		return nil
+	}
+	acc.closed = true
+	close(acc.exit)
+	Stats.inProgress.clear(acc.name)
+	return acc.close.Close()
+}
+
+// progress returns bytes read as well as the size.
+// Size can be <= 0 if the size is unknown.
+func (acc *Account) progress() (bytes, size int64) {
+	if acc == nil {
+		return 0, 0
+	}
+	acc.statmu.Lock()
+	bytes, size = acc.bytes, acc.size
+	acc.statmu.Unlock()
+	return bytes, size
+}
+
+// speed returns the speed of the current file transfer
+// in bytes per second, as well a an exponentially weighted moving average
+// If no read has completed yet, 0 is returned for both values.
+func (acc *Account) speed() (bps, current float64) {
+	if acc == nil {
+		return 0, 0
+	}
+	acc.statmu.Lock()
+	defer acc.statmu.Unlock()
+	if acc.bytes == 0 {
+		return 0, 0
+	}
+	// Calculate speed from first read.
+	total := float64(time.Now().Sub(acc.start)) / float64(time.Second)
+	bps = float64(acc.bytes) / total
+	current = acc.avg
+	return
+}
+
+// eta returns the ETA of the current operation,
+// rounded to full seconds.
+// If the ETA cannot be determined 'ok' returns false.
+func (acc *Account) eta() (etaDuration time.Duration, ok bool) {
+	if acc == nil {
+		return 0, false
+	}
+	acc.statmu.Lock()
+	defer acc.statmu.Unlock()
+	return eta(acc.bytes, acc.size, acc.avg)
+}
+
+// String produces stats for this file
+func (acc *Account) String() string {
+	a, b := acc.progress()
+	_, cur := acc.speed()
+	eta, etaok := acc.eta()
+	etas := "-"
+	if etaok {
+		if eta > 0 {
+			etas = fmt.Sprintf("%v", eta)
+		} else {
+			etas = "0s"
+		}
+	}
+	name := []rune(acc.name)
+	if fs.Config.StatsFileNameLength > 0 {
+		if len(name) > fs.Config.StatsFileNameLength {
+			where := len(name) - fs.Config.StatsFileNameLength
+			name = append([]rune{'.', '.', '.'}, name[where:]...)
+		}
+	}
+
+	if fs.Config.DataRateUnit == "bits" {
+		cur = cur * 8
+	}
+
+	percentageDone := 0
+	if b > 0 {
+		percentageDone = int(100 * float64(a) / float64(b))
+	}
+
+	done := fmt.Sprintf("%2d%% /%s", percentageDone, fs.SizeSuffix(b))
+
+	return fmt.Sprintf("%45s: %s, %s/s, %s",
+		string(name),
+		done,
+		fs.SizeSuffix(cur),
+		etas,
+	)
+}
+
+// RemoteStats produces stats for this file
+func (acc *Account) RemoteStats() (out map[string]interface{}) {
+	out = make(map[string]interface{})
+	a, b := acc.progress()
+	out["bytes"] = a
+	out["size"] = b
+	spd, cur := acc.speed()
+	out["speed"] = spd
+	out["speedAvg"] = cur
+
+	eta, etaok := acc.eta()
+	out["eta"] = nil
+	if etaok {
+		if eta > 0 {
+			out["eta"] = eta.Seconds()
+		} else {
+			out["eta"] = 0
+		}
+	}
+	out["name"] = acc.name
+
+	percentageDone := 0
+	if b > 0 {
+		percentageDone = int(100 * float64(a) / float64(b))
+	}
+	out["percentage"] = percentageDone
+
+	return out
+}
+
+// OldStream returns the top io.Reader
+func (acc *Account) OldStream() io.Reader {
+	acc.mu.Lock()
+	defer acc.mu.Unlock()
+	return acc.in
+}
+
+// SetStream updates the top io.Reader
+func (acc *Account) SetStream(in io.Reader) {
+	acc.mu.Lock()
+	acc.in = in
+	acc.mu.Unlock()
+}
+
+// WrapStream wraps an io Reader so it will be accounted in the same
+// way as account
+func (acc *Account) WrapStream(in io.Reader) io.Reader {
+	return &accountStream{
+		acc: acc,
+		in:  in,
+	}
+}
+
+// accountStream accounts a single io.Reader into a parent *Account
+type accountStream struct {
+	acc *Account
+	in  io.Reader
+}
+
+// OldStream return the underlying stream
+func (a *accountStream) OldStream() io.Reader {
+	return a.in
+}
+
+// SetStream set the underlying stream
+func (a *accountStream) SetStream(in io.Reader) {
+	a.in = in
+}
+
+// WrapStream wrap in in an accounter
+func (a *accountStream) WrapStream(in io.Reader) io.Reader {
+	return a.acc.WrapStream(in)
+}
+
+// Read bytes from the object - see io.Reader
+func (a *accountStream) Read(p []byte) (n int, err error) {
+	return a.acc.read(a.in, p)
+}
+
+// Accounter accounts a stream allowing the accounting to be removed and re-added
+type Accounter interface {
+	io.Reader
+	OldStream() io.Reader
+	SetStream(io.Reader)
+	WrapStream(io.Reader) io.Reader
+}
+
+// WrapFn wraps an io.Reader (for accounting purposes usually)
+type WrapFn func(io.Reader) io.Reader
+
+// UnWrap unwraps a reader returning unwrapped and wrap, a function to
+// wrap it back up again.  If `in` is an Accounter then this function
+// will take the accounting unwrapped and wrap will put it back on
+// again the new Reader passed in.
+//
+// This allows functions which wrap io.Readers to move the accounting
+// to the end of the wrapped chain of readers.  This is very important
+// if buffering is being introduced and if the Reader might be wrapped
+// again.
+func UnWrap(in io.Reader) (unwrapped io.Reader, wrap WrapFn) {
+	acc, ok := in.(Accounter)
+	if !ok {
+		return in, func(r io.Reader) io.Reader { return r }
+	}
+	return acc.OldStream(), acc.WrapStream
+}

vendor/github.com/ncw/rclone/fs/accounting/accounting_other.go

@@ -0,0 +1,10 @@
+// Accounting and limiting reader
+// Non-unix specific functions.
+
+// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris
+
+package accounting
+
+// startSignalHandler() is Unix specific and does nothing under non-Unix
+// platforms.
+func startSignalHandler() {}

vendor/github.com/ncw/rclone/fs/accounting/accounting_unix.go

@@ -0,0 +1,36 @@
+// Accounting and limiting reader
+// Unix specific functions.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd solaris
+
+package accounting
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/ncw/rclone/fs"
+)
+
+// startSignalHandler() sets a signal handler to catch SIGUSR2 and toggle throttling.
+func startSignalHandler() {
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, syscall.SIGUSR2)
+
+	go func() {
+		// This runs forever, but blocks until the signal is received.
+		for {
+			<-signals
+			tokenBucketMu.Lock()
+			bwLimitToggledOff = !bwLimitToggledOff
+			tokenBucket, prevTokenBucket = prevTokenBucket, tokenBucket
+			s := "disabled"
+			if tokenBucket != nil {
+				s = "enabled"
+			}
+			tokenBucketMu.Unlock()
+			fs.Logf(nil, "Bandwidth limit %s by user", s)
+		}
+	}()
+}

vendor/github.com/ncw/rclone/fs/accounting/inprogress.go

@@ -0,0 +1,41 @@
+package accounting
+
+import (
+	"sync"
+
+	"github.com/ncw/rclone/fs"
+)
+
+// inProgress holds a synchronized map of in progress transfers
+type inProgress struct {
+	mu sync.Mutex
+	m  map[string]*Account
+}
+
+// newInProgress makes a new inProgress object
+func newInProgress() *inProgress {
+	return &inProgress{
+		m: make(map[string]*Account, fs.Config.Transfers),
+	}
+}
+
+// set marks the name as in progress
+func (ip *inProgress) set(name string, acc *Account) {
+	ip.mu.Lock()
+	defer ip.mu.Unlock()
+	ip.m[name] = acc
+}
+
+// clear marks the name as no longer in progress
+func (ip *inProgress) clear(name string) {
+	ip.mu.Lock()
+	defer ip.mu.Unlock()
+	delete(ip.m, name)
+}
+
+// get gets the account for name, of nil if not found
+func (ip *inProgress) get(name string) *Account {
+	ip.mu.Lock()
+	defer ip.mu.Unlock()
+	return ip.m[name]
+}

vendor/github.com/ncw/rclone/fs/accounting/stats.go

@@ -0,0 +1,406 @@
+package accounting
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/rc"
+)
+
+var (
+	// Stats is global statistics counter
+	Stats = NewStats()
+)
+
+func init() {
+	// Set the function pointer up in fs
+	fs.CountError = Stats.Error
+
+	rc.Add(rc.Call{
+		Path:  "core/stats",
+		Fn:    Stats.RemoteStats,
+		Title: "Returns stats about current transfers.",
+		Help: `
+This returns all available stats
+
+	rclone rc core/stats
+
+Returns the following values:
+
+` + "```" + `
+{
+	"speed": average speed in bytes/sec since start of the process,
+	"bytes": total transferred bytes since the start of the process,
+	"errors": number of errors,
+	"checks": number of checked files,
+	"transfers": number of transferred files,
+	"deletes" : number of deleted files,
+	"elapsedTime": time in seconds since the start of the process,
+	"lastError": last occurred error,
+	"transferring": an array of currently active file transfers:
+		[
+			{
+				"bytes": total transferred bytes for this file,
+				"eta": estimated time in seconds until file transfer completion
+				"name": name of the file,
+				"percentage": progress of the file transfer in percent,
+				"speed": speed in bytes/sec,
+				"speedAvg": speed in bytes/sec as an exponentially weighted moving average,
+				"size": size of the file in bytes
+			}
+		],
+	"checking": an array of names of currently active file checks
+		[]
+}
+` + "```" + `
+Values for "transferring", "checking" and "lastError" are only assigned if data is available.
+The value for "eta" is null if an eta cannot be determined.
+`,
+	})
+}
+
+// StatsInfo accounts all transfers
+type StatsInfo struct {
+	mu                sync.RWMutex
+	bytes             int64
+	errors            int64
+	lastError         error
+	checks            int64
+	checking          *stringSet
+	checkQueue        int
+	checkQueueSize    int64
+	transfers         int64
+	transferring      *stringSet
+	transferQueue     int
+	transferQueueSize int64
+	renameQueue       int
+	renameQueueSize   int64
+	deletes           int64
+	start             time.Time
+	inProgress        *inProgress
+}
+
+// NewStats cretates an initialised StatsInfo
+func NewStats() *StatsInfo {
+	return &StatsInfo{
+		checking:     newStringSet(fs.Config.Checkers),
+		transferring: newStringSet(fs.Config.Transfers),
+		start:        time.Now(),
+		inProgress:   newInProgress(),
+	}
+}
+
+// RemoteStats returns stats for rc
+func (s *StatsInfo) RemoteStats(in rc.Params) (out rc.Params, err error) {
+	out = make(rc.Params)
+	s.mu.RLock()
+	dt := time.Now().Sub(s.start)
+	dtSeconds := dt.Seconds()
+	speed := 0.0
+	if dt > 0 {
+		speed = float64(s.bytes) / dtSeconds
+	}
+	out["speed"] = speed
+	out["bytes"] = s.bytes
+	out["errors"] = s.errors
+	out["checks"] = s.checks
+	out["transfers"] = s.transfers
+	out["deletes"] = s.deletes
+	out["elapsedTime"] = dtSeconds
+	s.mu.RUnlock()
+	if !s.checking.empty() {
+		var c []string
+		s.checking.mu.RLock()
+		defer s.checking.mu.RUnlock()
+		for name := range s.checking.items {
+			c = append(c, name)
+		}
+		out["checking"] = c
+	}
+	if !s.transferring.empty() {
+		var t []interface{}
+		s.transferring.mu.RLock()
+		defer s.transferring.mu.RUnlock()
+		for name := range s.transferring.items {
+			if acc := s.inProgress.get(name); acc != nil {
+				t = append(t, acc.RemoteStats())
+			} else {
+				t = append(t, name)
+			}
+		}
+		out["transferring"] = t
+	}
+	if s.errors > 0 {
+		out["lastError"] = s.lastError
+	}
+	return out, nil
+}
+
+// eta returns the ETA of the current operation,
+// rounded to full seconds.
+// If the ETA cannot be determined 'ok' returns false.
+func eta(size, total int64, rate float64) (eta time.Duration, ok bool) {
+	if total <= 0 || size < 0 || rate <= 0 {
+		return 0, false
+	}
+	remaining := total - size
+	if remaining < 0 {
+		return 0, false
+	}
+	seconds := float64(remaining) / rate
+	return time.Second * time.Duration(seconds), true
+}
+
+// etaString returns the ETA of the current operation,
+// rounded to full seconds.
+// If the ETA cannot be determined it returns "-"
+func etaString(done, total int64, rate float64) string {
+	d, ok := eta(done, total, rate)
+	if !ok {
+		return "-"
+	}
+	return d.String()
+}
+
+// percent returns a/b as a percentage rounded to the nearest integer
+// as a string
+//
+// if the percentage is invalid it returns "-"
+func percent(a int64, b int64) string {
+	if a < 0 || b <= 0 {
+		return "-"
+	}
+	return fmt.Sprintf("%d%%", int(float64(a)*100/float64(b)+0.5))
+}
+
+// String convert the StatsInfo to a string for printing
+func (s *StatsInfo) String() string {
+	// checking and transferring have their own locking so read
+	// here before lock to prevent deadlock on GetBytes
+	transferring, checking := s.transferring.count(), s.checking.count()
+	transferringBytesDone, transferringBytesTotal := s.transferring.progress()
+
+	s.mu.RLock()
+
+	dt := time.Now().Sub(s.start)
+	dtSeconds := dt.Seconds()
+	speed := 0.0
+	if dt > 0 {
+		speed = float64(s.bytes) / dtSeconds
+	}
+	dtRounded := dt - (dt % (time.Second / 10))
+
+	if fs.Config.DataRateUnit == "bits" {
+		speed = speed * 8
+	}
+
+	var (
+		totalChecks   = int64(s.checkQueue) + s.checks + int64(checking)
+		totalTransfer = int64(s.transferQueue) + s.transfers + int64(transferring)
+		// note that s.bytes already includes transferringBytesDone so
+		// we take it off here to avoid double counting
+		totalSize    = s.transferQueueSize + s.bytes + transferringBytesTotal - transferringBytesDone
+		currentSize  = s.bytes
+		buf          = &bytes.Buffer{}
+		xfrchkString = ""
+	)
+
+	if !fs.Config.StatsOneLine {
+		_, _ = fmt.Fprintf(buf, "\nTransferred:   	")
+	} else {
+		xfrchk := []string{}
+		if totalTransfer > 0 && s.transferQueue > 0 {
+			xfrchk = append(xfrchk, fmt.Sprintf("xfr#%d/%d", s.transfers, totalTransfer))
+		}
+		if totalChecks > 0 && s.checkQueue > 0 {
+			xfrchk = append(xfrchk, fmt.Sprintf("chk#%d/%d", s.checks, totalChecks))
+		}
+		if len(xfrchk) > 0 {
+			xfrchkString = fmt.Sprintf(" (%s)", strings.Join(xfrchk, ", "))
+		}
+	}
+
+	_, _ = fmt.Fprintf(buf, "%10s / %s, %s, %s, ETA %s%s",
+		fs.SizeSuffix(s.bytes),
+		fs.SizeSuffix(totalSize).Unit("Bytes"),
+		percent(s.bytes, totalSize),
+		fs.SizeSuffix(speed).Unit(strings.Title(fs.Config.DataRateUnit)+"/s"),
+		etaString(currentSize, totalSize, speed),
+		xfrchkString,
+	)
+
+	if !fs.Config.StatsOneLine {
+		_, _ = fmt.Fprintf(buf, `
+Errors:        %10d
+Checks:        %10d / %d, %s
+Transferred:   %10d / %d, %s
+Elapsed time:  %10v
+`,
+			s.errors,
+			s.checks, totalChecks, percent(s.checks, totalChecks),
+			s.transfers, totalTransfer, percent(s.transfers, totalTransfer),
+			dtRounded)
+	}
+
+	// checking and transferring have their own locking so unlock
+	// here to prevent deadlock on GetBytes
+	s.mu.RUnlock()
+
+	// Add per transfer stats if required
+	if !fs.Config.StatsOneLine {
+		if !s.checking.empty() {
+			_, _ = fmt.Fprintf(buf, "Checking:\n%s\n", s.checking)
+		}
+		if !s.transferring.empty() {
+			_, _ = fmt.Fprintf(buf, "Transferring:\n%s\n", s.transferring)
+		}
+	}
+
+	return buf.String()
+}
+
+// Log outputs the StatsInfo to the log
+func (s *StatsInfo) Log() {
+	fs.LogLevelPrintf(fs.Config.StatsLogLevel, nil, "%v\n", s)
+}
+
+// Bytes updates the stats for bytes bytes
+func (s *StatsInfo) Bytes(bytes int64) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.bytes += bytes
+}
+
+// GetBytes returns the number of bytes transferred so far
+func (s *StatsInfo) GetBytes() int64 {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.bytes
+}
+
+// Errors updates the stats for errors
+func (s *StatsInfo) Errors(errors int64) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.errors += errors
+}
+
+// GetErrors reads the number of errors
+func (s *StatsInfo) GetErrors() int64 {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.errors
+}
+
+// GetLastError returns the lastError
+func (s *StatsInfo) GetLastError() error {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.lastError
+}
+
+// Deletes updates the stats for deletes
+func (s *StatsInfo) Deletes(deletes int64) int64 {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.deletes += deletes
+	return s.deletes
+}
+
+// ResetCounters sets the counters (bytes, checks, errors, transfers) to 0
+func (s *StatsInfo) ResetCounters() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.bytes = 0
+	s.errors = 0
+	s.checks = 0
+	s.transfers = 0
+	s.deletes = 0
+}
+
+// ResetErrors sets the errors count to 0
+func (s *StatsInfo) ResetErrors() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.errors = 0
+}
+
+// Errored returns whether there have been any errors
+func (s *StatsInfo) Errored() bool {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.errors != 0
+}
+
+// Error adds a single error into the stats and assigns lastError
+func (s *StatsInfo) Error(err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.errors++
+	s.lastError = err
+}
+
+// Checking adds a check into the stats
+func (s *StatsInfo) Checking(remote string) {
+	s.checking.add(remote)
+}
+
+// DoneChecking removes a check from the stats
+func (s *StatsInfo) DoneChecking(remote string) {
+	s.checking.del(remote)
+	s.mu.Lock()
+	s.checks++
+	s.mu.Unlock()
+}
+
+// GetTransfers reads the number of transfers
+func (s *StatsInfo) GetTransfers() int64 {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.transfers
+}
+
+// Transferring adds a transfer into the stats
+func (s *StatsInfo) Transferring(remote string) {
+	s.transferring.add(remote)
+}
+
+// DoneTransferring removes a transfer from the stats
+//
+// if ok is true then it increments the transfers count
+func (s *StatsInfo) DoneTransferring(remote string, ok bool) {
+	s.transferring.del(remote)
+	if ok {
+		s.mu.Lock()
+		s.transfers++
+		s.mu.Unlock()
+	}
+}
+
+// SetCheckQueue sets the number of queued checks
+func (s *StatsInfo) SetCheckQueue(n int, size int64) {
+	s.mu.Lock()
+	s.checkQueue = n
+	s.checkQueueSize = size
+	s.mu.Unlock()
+}
+
+// SetTransferQueue sets the number of queued transfers
+func (s *StatsInfo) SetTransferQueue(n int, size int64) {
+	s.mu.Lock()
+	s.transferQueue = n
+	s.transferQueueSize = size
+	s.mu.Unlock()
+}
+
+// SetRenameQueue sets the number of queued transfers
+func (s *StatsInfo) SetRenameQueue(n int, size int64) {
+	s.mu.Lock()
+	s.renameQueue = n
+	s.renameQueueSize = size
+	s.mu.Unlock()
+}

vendor/github.com/ncw/rclone/fs/accounting/stringset.go

@@ -0,0 +1,88 @@
+package accounting
+
+import (
+	"sort"
+	"strings"
+	"sync"
+)
+
+// stringSet holds a set of strings
+type stringSet struct {
+	mu    sync.RWMutex
+	items map[string]struct{}
+}
+
+// newStringSet creates a new empty string set of capacity size
+func newStringSet(size int) *stringSet {
+	return &stringSet{
+		items: make(map[string]struct{}, size),
+	}
+}
+
+// add adds remote to the set
+func (ss *stringSet) add(remote string) {
+	ss.mu.Lock()
+	ss.items[remote] = struct{}{}
+	ss.mu.Unlock()
+}
+
+// del removes remote from the set
+func (ss *stringSet) del(remote string) {
+	ss.mu.Lock()
+	delete(ss.items, remote)
+	ss.mu.Unlock()
+}
+
+// empty returns whether the set has any items
+func (ss *stringSet) empty() bool {
+	ss.mu.RLock()
+	defer ss.mu.RUnlock()
+	return len(ss.items) == 0
+}
+
+// count returns the number of items in the set
+func (ss *stringSet) count() int {
+	ss.mu.RLock()
+	defer ss.mu.RUnlock()
+	return len(ss.items)
+}
+
+// Strings returns all the strings in the stringSet
+func (ss *stringSet) Strings() []string {
+	ss.mu.RLock()
+	defer ss.mu.RUnlock()
+	strings := make([]string, 0, len(ss.items))
+	for name := range ss.items {
+		var out string
+		if acc := Stats.inProgress.get(name); acc != nil {
+			out = acc.String()
+		} else {
+			out = name
+		}
+		strings = append(strings, " * "+out)
+	}
+	sorted := sort.StringSlice(strings)
+	sorted.Sort()
+	return sorted
+}
+
+// String returns all the file names in the stringSet joined by newline
+func (ss *stringSet) String() string {
+	return strings.Join(ss.Strings(), "\n")
+}
+
+// progress returns total bytes read as well as the size.
+func (ss *stringSet) progress() (totalBytes, totalSize int64) {
+	ss.mu.RLock()
+	defer ss.mu.RUnlock()
+	for name := range ss.items {
+		if acc := Stats.inProgress.get(name); acc != nil {
+			bytes, size := acc.progress()
+			if size >= 0 && bytes >= 0 {
+				totalBytes += bytes
+				totalSize += size
+			}
+		}
+	}
+	return totalBytes, totalSize
+}

vendor/github.com/ncw/rclone/fs/accounting/token_bucket.go

@@ -0,0 +1,169 @@
+package accounting
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/rc"
+	"github.com/pkg/errors"
+	"golang.org/x/time/rate"
+)
+
+// Globals
+var (
+	tokenBucketMu     sync.Mutex // protects the token bucket variables
+	tokenBucket       *rate.Limiter
+	prevTokenBucket   = tokenBucket
+	bwLimitToggledOff = false
+	currLimitMu       sync.Mutex // protects changes to the timeslot
+	currLimit         fs.BwTimeSlot
+)
+
+const maxBurstSize = 4 * 1024 * 1024 // must be bigger than the biggest request
+
+// make a new empty token bucket with the bandwidth given
+func newTokenBucket(bandwidth fs.SizeSuffix) *rate.Limiter {
+	newTokenBucket := rate.NewLimiter(rate.Limit(bandwidth), maxBurstSize)
+	// empty the bucket
+	err := newTokenBucket.WaitN(context.Background(), maxBurstSize)
+	if err != nil {
+		fs.Errorf(nil, "Failed to empty token bucket: %v", err)
+	}
+	return newTokenBucket
+}
+
+// StartTokenBucket starts the token bucket if necessary
+func StartTokenBucket() {
+	currLimitMu.Lock()
+	currLimit := fs.Config.BwLimit.LimitAt(time.Now())
+	currLimitMu.Unlock()
+
+	if currLimit.Bandwidth > 0 {
+		tokenBucket = newTokenBucket(currLimit.Bandwidth)
+		fs.Infof(nil, "Starting bandwidth limiter at %vBytes/s", &currLimit.Bandwidth)
+
+		// Start the SIGUSR2 signal handler to toggle bandwidth.
+		// This function does nothing in windows systems.
+		startSignalHandler()
+	}
+}
+
+// StartTokenTicker creates a ticker to update the bandwidth limiter every minute.
+func StartTokenTicker() {
+	// If the timetable has a single entry or was not specified, we don't need
+	// a ticker to update the bandwidth.
+	if len(fs.Config.BwLimit) <= 1 {
+		return
+	}
+
+	ticker := time.NewTicker(time.Minute)
+	go func() {
+		for range ticker.C {
+			limitNow := fs.Config.BwLimit.LimitAt(time.Now())
+			currLimitMu.Lock()
+
+			if currLimit.Bandwidth != limitNow.Bandwidth {
+				tokenBucketMu.Lock()
+
+				// If bwlimit is toggled off, the change should only
+				// become active on the next toggle, which causes
+				// an exchange of tokenBucket <-> prevTokenBucket
+				var targetBucket **rate.Limiter
+				if bwLimitToggledOff {
+					targetBucket = &prevTokenBucket
+				} else {
+					targetBucket = &tokenBucket
+				}
+
+				// Set new bandwidth. If unlimited, set tokenbucket to nil.
+				if limitNow.Bandwidth > 0 {
+					*targetBucket = newTokenBucket(limitNow.Bandwidth)
+					if bwLimitToggledOff {
+						fs.Logf(nil, "Scheduled bandwidth change. "+
+							"Limit will be set to %vBytes/s when toggled on again.", &limitNow.Bandwidth)
+					} else {
+						fs.Logf(nil, "Scheduled bandwidth change. Limit set to %vBytes/s", &limitNow.Bandwidth)
+					}
+				} else {
+					*targetBucket = nil
+					fs.Logf(nil, "Scheduled bandwidth change. Bandwidth limits disabled")
+				}
+
+				currLimit = limitNow
+				tokenBucketMu.Unlock()
+			}
+			currLimitMu.Unlock()
+		}
+	}()
+}
+
+// limitBandwith sleeps for the correct amount of time for the passage
+// of n bytes according to the current bandwidth limit
+func limitBandwidth(n int) {
+	tokenBucketMu.Lock()
+
+	// Limit the transfer speed if required
+	if tokenBucket != nil {
+		err := tokenBucket.WaitN(context.Background(), n)
+		if err != nil {
+			fs.Errorf(nil, "Token bucket error: %v", err)
+		}
+	}
+
+	tokenBucketMu.Unlock()
+}
+
+// SetBwLimit sets the current bandwidth limit
+func SetBwLimit(bandwidth fs.SizeSuffix) {
+	tokenBucketMu.Lock()
+	defer tokenBucketMu.Unlock()
+	if bandwidth > 0 {
+		tokenBucket = newTokenBucket(bandwidth)
+		fs.Logf(nil, "Bandwidth limit set to %v", bandwidth)
+	} else {
+		tokenBucket = nil
+		fs.Logf(nil, "Bandwidth limit reset to unlimited")
+	}
+}
+
+// Remote control for the token bucket
+func init() {
+	rc.Add(rc.Call{
+		Path: "core/bwlimit",
+		Fn: func(in rc.Params) (out rc.Params, err error) {
+			ibwlimit, ok := in["rate"]
+			if !ok {
+				return out, errors.Errorf("parameter rate not found")
+			}
+			bwlimit, ok := ibwlimit.(string)
+			if !ok {
+				return out, errors.Errorf("value must be string rate=%v", ibwlimit)
+			}
+			var bws fs.BwTimetable
+			err = bws.Set(bwlimit)
+			if err != nil {
+				return out, errors.Wrap(err, "bad bwlimit")
+			}
+			if len(bws) != 1 {
+				return out, errors.New("need exactly 1 bandwidth setting")
+			}
+			bw := bws[0]
+			SetBwLimit(bw.Bandwidth)
+			return rc.Params{"rate": bw.Bandwidth.String()}, nil
+		},
+		Title: "Set the bandwidth limit.",
+		Help: `
+This sets the bandwidth limit to that passed in.
+
+Eg
+
+    rclone rc core/bwlimit rate=1M
+    rclone rc core/bwlimit rate=off
+
+The format of the parameter is exactly the same as passed to --bwlimit
+except only one bandwidth may be specified.
+`,
+	})
+}

vendor/github.com/ncw/rclone/fs/asyncreader/asyncreader.go

@@ -0,0 +1,343 @@
+// Package asyncreader provides an asynchronous reader which reads
+// independently of write
+package asyncreader
+
+import (
+	"io"
+	"sync"
+
+	"github.com/ncw/rclone/lib/readers"
+	"github.com/pkg/errors"
+)
+
+const (
+	// BufferSize is the default size of the async buffer
+	BufferSize       = 1024 * 1024
+	softStartInitial = 4 * 1024
+)
+
+var asyncBufferPool = sync.Pool{
+	New: func() interface{} { return newBuffer() },
+}
+
+var errorStreamAbandoned = errors.New("stream abandoned")
+
+// AsyncReader will do async read-ahead from the input reader
+// and make the data available as an io.Reader.
+// This should be fully transparent, except that once an error
+// has been returned from the Reader, it will not recover.
+type AsyncReader struct {
+	in      io.ReadCloser // Input reader
+	ready   chan *buffer  // Buffers ready to be handed to the reader
+	token   chan struct{} // Tokens which allow a buffer to be taken
+	exit    chan struct{} // Closes when finished
+	buffers int           // Number of buffers
+	err     error         // If an error has occurred it is here
+	cur     *buffer       // Current buffer being served
+	exited  chan struct{} // Channel is closed been the async reader shuts down
+	size    int           // size of buffer to use
+	closed  bool          // whether we have closed the underlying stream
+	mu      sync.Mutex    // lock for Read/WriteTo/Abandon/Close
+}
+
+// New returns a reader that will asynchronously read from
+// the supplied Reader into a number of buffers each of size BufferSize
+// It will start reading from the input at once, maybe even before this
+// function has returned.
+// The input can be read from the returned reader.
+// When done use Close to release the buffers and close the supplied input.
+func New(rd io.ReadCloser, buffers int) (*AsyncReader, error) {
+	if buffers <= 0 {
+		return nil, errors.New("number of buffers too small")
+	}
+	if rd == nil {
+		return nil, errors.New("nil reader supplied")
+	}
+	a := &AsyncReader{}
+	a.init(rd, buffers)
+	return a, nil
+}
+
+func (a *AsyncReader) init(rd io.ReadCloser, buffers int) {
+	a.in = rd
+	a.ready = make(chan *buffer, buffers)
+	a.token = make(chan struct{}, buffers)
+	a.exit = make(chan struct{}, 0)
+	a.exited = make(chan struct{}, 0)
+	a.buffers = buffers
+	a.cur = nil
+	a.size = softStartInitial
+
+	// Create tokens
+	for i := 0; i < buffers; i++ {
+		a.token <- struct{}{}
+	}
+
+	// Start async reader
+	go func() {
+		// Ensure that when we exit this is signalled.
+		defer close(a.exited)
+		defer close(a.ready)
+		for {
+			select {
+			case <-a.token:
+				b := a.getBuffer()
+				if a.size < BufferSize {
+					b.buf = b.buf[:a.size]
+					a.size <<= 1
+				}
+				err := b.read(a.in)
+				a.ready <- b
+				if err != nil {
+					return
+				}
+			case <-a.exit:
+				return
+			}
+		}
+	}()
+}
+
+// return the buffer to the pool (clearing it)
+func (a *AsyncReader) putBuffer(b *buffer) {
+	b.clear()
+	asyncBufferPool.Put(b)
+}
+
+// get a buffer from the pool
+func (a *AsyncReader) getBuffer() *buffer {
+	b := asyncBufferPool.Get().(*buffer)
+	return b
+}
+
+// Read will return the next available data.
+func (a *AsyncReader) fill() (err error) {
+	if a.cur.isEmpty() {
+		if a.cur != nil {
+			a.putBuffer(a.cur)
+			a.token <- struct{}{}
+			a.cur = nil
+		}
+		b, ok := <-a.ready
+		if !ok {
+			// Return an error to show fill failed
+			if a.err == nil {
+				return errorStreamAbandoned
+			}
+			return a.err
+		}
+		a.cur = b
+	}
+	return nil
+}
+
+// Read will return the next available data.
+func (a *AsyncReader) Read(p []byte) (n int, err error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	// Swap buffer and maybe return error
+	err = a.fill()
+	if err != nil {
+		return 0, err
+	}
+
+	// Copy what we can
+	n = copy(p, a.cur.buffer())
+	a.cur.increment(n)
+
+	// If at end of buffer, return any error, if present
+	if a.cur.isEmpty() {
+		a.err = a.cur.err
+		return n, a.err
+	}
+	return n, nil
+}
+
+// WriteTo writes data to w until there's no more data to write or when an error occurs.
+// The return value n is the number of bytes written.
+// Any error encountered during the write is also returned.
+func (a *AsyncReader) WriteTo(w io.Writer) (n int64, err error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	n = 0
+	for {
+		err = a.fill()
+		if err != nil {
+			return n, err
+		}
+		n2, err := w.Write(a.cur.buffer())
+		a.cur.increment(n2)
+		n += int64(n2)
+		if err != nil {
+			return n, err
+		}
+		if a.cur.err != nil {
+			a.err = a.cur.err
+			return n, a.cur.err
+		}
+	}
+}
+
+// SkipBytes will try to seek 'skip' bytes relative to the current position.
+// On success it returns true. If 'skip' is outside the current buffer data or
+// an error occurs, Abandon is called and false is returned.
+func (a *AsyncReader) SkipBytes(skip int) (ok bool) {
+	a.mu.Lock()
+	defer func() {
+		a.mu.Unlock()
+		if !ok {
+			a.Abandon()
+		}
+	}()
+
+	if a.err != nil {
+		return false
+	}
+	if skip < 0 {
+		// seek backwards if skip is inside current buffer
+		if a.cur != nil && a.cur.offset+skip >= 0 {
+			a.cur.offset += skip
+			return true
+		}
+		return false
+	}
+	// early return if skip is past the maximum buffer capacity
+	if skip >= (len(a.ready)+1)*BufferSize {
+		return false
+	}
+
+	refillTokens := 0
+	for {
+		if a.cur.isEmpty() {
+			if a.cur != nil {
+				a.putBuffer(a.cur)
+				refillTokens++
+				a.cur = nil
+			}
+			select {
+			case b, ok := <-a.ready:
+				if !ok {
+					return false
+				}
+				a.cur = b
+			default:
+				return false
+			}
+		}
+
+		n := len(a.cur.buffer())
+		if n > skip {
+			n = skip
+		}
+		a.cur.increment(n)
+		skip -= n
+		if skip == 0 {
+			for ; refillTokens > 0; refillTokens-- {
+				a.token <- struct{}{}
+			}
+			// If at end of buffer, store any error, if present
+			if a.cur.isEmpty() && a.cur.err != nil {
+				a.err = a.cur.err
+			}
+			return true
+		}
+		if a.cur.err != nil {
+			a.err = a.cur.err
+			return false
+		}
+	}
+}
+
+// Abandon will ensure that the underlying async reader is shut down.
+// It will NOT close the input supplied on New.
+func (a *AsyncReader) Abandon() {
+	select {
+	case <-a.exit:
+		// Do nothing if reader routine already exited
+		return
+	default:
+	}
+	// Close and wait for go routine
+	close(a.exit)
+	<-a.exited
+	// take the lock to wait for Read/WriteTo to complete
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	// Return any outstanding buffers to the Pool
+	if a.cur != nil {
+		a.putBuffer(a.cur)
+		a.cur = nil
+	}
+	for b := range a.ready {
+		a.putBuffer(b)
+	}
+}
+
+// Close will ensure that the underlying async reader is shut down.
+// It will also close the input supplied on New.
+func (a *AsyncReader) Close() (err error) {
+	a.Abandon()
+	if a.closed {
+		return nil
+	}
+	a.closed = true
+	return a.in.Close()
+}
+
+// Internal buffer
+// If an error is present, it must be returned
+// once all buffer content has been served.
+type buffer struct {
+	buf    []byte
+	err    error
+	offset int
+}
+
+func newBuffer() *buffer {
+	return &buffer{
+		buf: make([]byte, BufferSize),
+		err: nil,
+	}
+}
+
+// clear returns the buffer to its full size and clears the members
+func (b *buffer) clear() {
+	b.buf = b.buf[:cap(b.buf)]
+	b.err = nil
+	b.offset = 0
+}
+
+// isEmpty returns true is offset is at end of
+// buffer, or
+func (b *buffer) isEmpty() bool {
+	if b == nil {
+		return true
+	}
+	if len(b.buf)-b.offset <= 0 {
+		return true
+	}
+	return false
+}
+
+// read into start of the buffer from the supplied reader,
+// resets the offset and updates the size of the buffer.
+// Any error encountered during the read is returned.
+func (b *buffer) read(rd io.Reader) error {
+	var n int
+	n, b.err = readers.ReadFill(rd, b.buf)
+	b.buf = b.buf[0:n]
+	b.offset = 0
+	return b.err
+}
+
+// Return the buffer at current offset
+func (b *buffer) buffer() []byte {
+	return b.buf[b.offset:]
+}
+
+// increment the offset
+func (b *buffer) increment(n int) {
+	b.offset += n
+}

vendor/github.com/ncw/rclone/fs/bwtimetable.go

@@ -0,0 +1,214 @@
+package fs
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// BwTimeSlot represents a bandwidth configuration at a point in time.
+type BwTimeSlot struct {
+	DayOfTheWeek int
+	HHMM         int
+	Bandwidth    SizeSuffix
+}
+
+// BwTimetable contains all configured time slots.
+type BwTimetable []BwTimeSlot
+
+// String returns a printable representation of BwTimetable.
+func (x BwTimetable) String() string {
+	ret := []string{}
+	for _, ts := range x {
+		ret = append(ret, fmt.Sprintf("%s-%04.4d,%s", time.Weekday(ts.DayOfTheWeek), ts.HHMM, ts.Bandwidth.String()))
+	}
+	return strings.Join(ret, " ")
+}
+
+// Basic hour format checking
+func validateHour(HHMM string) error {
+	if len(HHMM) != 5 {
+		return errors.Errorf("invalid time specification (hh:mm): %q", HHMM)
+	}
+	hh, err := strconv.Atoi(HHMM[0:2])
+	if err != nil {
+		return errors.Errorf("invalid hour in time specification %q: %v", HHMM, err)
+	}
+	if hh < 0 || hh > 23 {
+		return errors.Errorf("invalid hour (must be between 00 and 23): %q", hh)
+	}
+	mm, err := strconv.Atoi(HHMM[3:])
+	if err != nil {
+		return errors.Errorf("invalid minute in time specification: %q: %v", HHMM, err)
+	}
+	if mm < 0 || mm > 59 {
+		return errors.Errorf("invalid minute (must be between 00 and 59): %q", hh)
+	}
+	return nil
+}
+
+// Basic weekday format checking
+func parseWeekday(dayOfWeek string) (int, error) {
+	dayOfWeek = strings.ToLower(dayOfWeek)
+	if dayOfWeek == "sun" || dayOfWeek == "sunday" {
+		return 0, nil
+	}
+	if dayOfWeek == "mon" || dayOfWeek == "monday" {
+		return 1, nil
+	}
+	if dayOfWeek == "tue" || dayOfWeek == "tuesday" {
+		return 2, nil
+	}
+	if dayOfWeek == "wed" || dayOfWeek == "wednesday" {
+		return 3, nil
+	}
+	if dayOfWeek == "thu" || dayOfWeek == "thursday" {
+		return 4, nil
+	}
+	if dayOfWeek == "fri" || dayOfWeek == "friday" {
+		return 5, nil
+	}
+	if dayOfWeek == "sat" || dayOfWeek == "saturday" {
+		return 6, nil
+	}
+	return 0, errors.Errorf("invalid weekday: %q", dayOfWeek)
+}
+
+// Set the bandwidth timetable.
+func (x *BwTimetable) Set(s string) error {
+	// The timetable is formatted as:
+	// "dayOfWeek-hh:mm,bandwidth dayOfWeek-hh:mm,banwidth..." ex: "Mon-10:00,10G Mon-11:30,1G Tue-18:00,off"
+	// If only a single bandwidth identifier is provided, we assume constant bandwidth.
+
+	if len(s) == 0 {
+		return errors.New("empty string")
+	}
+	// Single value without time specification.
+	if !strings.Contains(s, " ") && !strings.Contains(s, ",") {
+		ts := BwTimeSlot{}
+		if err := ts.Bandwidth.Set(s); err != nil {
+			return err
+		}
+		ts.DayOfTheWeek = 0
+		ts.HHMM = 0
+		*x = BwTimetable{ts}
+		return nil
+	}
+
+	for _, tok := range strings.Split(s, " ") {
+		tv := strings.Split(tok, ",")
+
+		// Format must be dayOfWeek-HH:MM,BW
+		if len(tv) != 2 {
+			return errors.Errorf("invalid time/bandwidth specification: %q", tok)
+		}
+
+		weekday := 0
+		HHMM := ""
+		if !strings.Contains(tv[0], "-") {
+			HHMM = tv[0]
+			if err := validateHour(HHMM); err != nil {
+				return err
+			}
+			for i := 0; i < 7; i++ {
+				hh, _ := strconv.Atoi(HHMM[0:2])
+				mm, _ := strconv.Atoi(HHMM[3:])
+				ts := BwTimeSlot{
+					DayOfTheWeek: i,
+					HHMM:         (hh * 100) + mm,
+				}
+				if err := ts.Bandwidth.Set(tv[1]); err != nil {
+					return err
+				}
+				*x = append(*x, ts)
+			}
+		} else {
+			timespec := strings.Split(tv[0], "-")
+			if len(timespec) != 2 {
+				return errors.Errorf("invalid time specification: %q", tv[0])
+			}
+			var err error
+			weekday, err = parseWeekday(timespec[0])
+			if err != nil {
+				return err
+			}
+			HHMM = timespec[1]
+			if err := validateHour(HHMM); err != nil {
+				return err
+			}
+
+			hh, _ := strconv.Atoi(HHMM[0:2])
+			mm, _ := strconv.Atoi(HHMM[3:])
+			ts := BwTimeSlot{
+				DayOfTheWeek: weekday,
+				HHMM:         (hh * 100) + mm,
+			}
+			// Bandwidth limit for this time slot.
+			if err := ts.Bandwidth.Set(tv[1]); err != nil {
+				return err
+			}
+			*x = append(*x, ts)
+		}
+	}
+	return nil
+}
+
+//	Difference in minutes between lateDayOfWeekHHMM and earlyDayOfWeekHHMM
+func timeDiff(lateDayOfWeekHHMM int, earlyDayOfWeekHHMM int) int {
+
+	lateTimeMinutes := (lateDayOfWeekHHMM / 10000) * 24 * 60
+	lateTimeMinutes += ((lateDayOfWeekHHMM / 100) % 100) * 60
+	lateTimeMinutes += lateDayOfWeekHHMM % 100
+
+	earlyTimeMinutes := (earlyDayOfWeekHHMM / 10000) * 24 * 60
+	earlyTimeMinutes += ((earlyDayOfWeekHHMM / 100) % 100) * 60
+	earlyTimeMinutes += earlyDayOfWeekHHMM % 100
+
+	return lateTimeMinutes - earlyTimeMinutes
+}
+
+// LimitAt returns a BwTimeSlot for the time requested.
+func (x BwTimetable) LimitAt(tt time.Time) BwTimeSlot {
+	// If the timetable is empty, we return an unlimited BwTimeSlot starting at Sunday midnight.
+	if len(x) == 0 {
+		return BwTimeSlot{DayOfTheWeek: 0, HHMM: 0, Bandwidth: -1}
+	}
+
+	dayOfWeekHHMM := int(tt.Weekday())*10000 + tt.Hour()*100 + tt.Minute()
+
+	// By default, we return the last element in the timetable. This
+	// satisfies two conditions: 1) If there's only one element it
+	// will always be selected, and 2) The last element of the table
+	// will "wrap around" until overridden by an earlier time slot.
+	// there's only one time slot in the timetable.
+	ret := x[len(x)-1]
+	mindif := 0
+	first := true
+
+	// Look for most recent time slot.
+	for _, ts := range x {
+		// Ignore the past
+		if dayOfWeekHHMM < (ts.DayOfTheWeek*10000)+ts.HHMM {
+			continue
+		}
+		dif := timeDiff(dayOfWeekHHMM, (ts.DayOfTheWeek*10000)+ts.HHMM)
+		if first {
+			mindif = dif
+			first = false
+		}
+		if dif <= mindif {
+			mindif = dif
+			ret = ts
+		}
+	}
+
+	return ret
+}
+
+// Type of the value
+func (x BwTimetable) Type() string {
+	return "BwTimetable"
+}

vendor/github.com/ncw/rclone/fs/config.go

@@ -0,0 +1,131 @@
+package fs
+
+import (
+	"net"
+	"strings"
+	"time"
+)
+
+// Global
+var (
+	// Config is the global config
+	Config = NewConfig()
+
+	// Read a value from the config file
+	//
+	// This is a function pointer to decouple the config
+	// implementation from the fs
+	ConfigFileGet = func(section, key string) (string, bool) { return "", false }
+
+	// Set a value into the config file
+	//
+	// This is a function pointer to decouple the config
+	// implementation from the fs
+	ConfigFileSet = func(section, key, value string) {
+		Errorf(nil, "No config handler to set %q = %q in section %q of the config file", key, value, section)
+	}
+
+	// CountError counts an error.  If any errors have been
+	// counted then it will exit with a non zero error code.
+	//
+	// This is a function pointer to decouple the config
+	// implementation from the fs
+	CountError = func(err error) {}
+
+	// ConfigProvider is the config key used for provider options
+	ConfigProvider = "provider"
+)
+
+// ConfigInfo is filesystem config options
+type ConfigInfo struct {
+	LogLevel              LogLevel
+	StatsLogLevel         LogLevel
+	DryRun                bool
+	CheckSum              bool
+	SizeOnly              bool
+	IgnoreTimes           bool
+	IgnoreExisting        bool
+	IgnoreErrors          bool
+	ModifyWindow          time.Duration
+	Checkers              int
+	Transfers             int
+	ConnectTimeout        time.Duration // Connect timeout
+	Timeout               time.Duration // Data channel timeout
+	Dump                  DumpFlags
+	InsecureSkipVerify    bool // Skip server certificate verification
+	DeleteMode            DeleteMode
+	MaxDelete             int64
+	TrackRenames          bool // Track file renames.
+	LowLevelRetries       int
+	UpdateOlder           bool // Skip files that are newer on the destination
+	NoGzip                bool // Disable compression
+	MaxDepth              int
+	IgnoreSize            bool
+	IgnoreChecksum        bool
+	NoUpdateModTime       bool
+	DataRateUnit          string
+	BackupDir             string
+	Suffix                string
+	UseListR              bool
+	BufferSize            SizeSuffix
+	BwLimit               BwTimetable
+	TPSLimit              float64
+	TPSLimitBurst         int
+	BindAddr              net.IP
+	DisableFeatures       []string
+	UserAgent             string
+	Immutable             bool
+	AutoConfirm           bool
+	StreamingUploadCutoff SizeSuffix
+	StatsFileNameLength   int
+	AskPassword           bool
+	UseServerModTime      bool
+	MaxTransfer           SizeSuffix
+	MaxBacklog            int
+	StatsOneLine          bool
+	Progress              bool
+}
+
+// NewConfig creates a new config with everything set to the default
+// value.  These are the ultimate defaults and are overriden by the
+// config module.
+func NewConfig() *ConfigInfo {
+	c := new(ConfigInfo)
+
+	// Set any values which aren't the zero for the type
+	c.LogLevel = LogLevelNotice
+	c.StatsLogLevel = LogLevelInfo
+	c.ModifyWindow = time.Nanosecond
+	c.Checkers = 8
+	c.Transfers = 4
+	c.ConnectTimeout = 60 * time.Second
+	c.Timeout = 5 * 60 * time.Second
+	c.DeleteMode = DeleteModeDefault
+	c.MaxDelete = -1
+	c.LowLevelRetries = 10
+	c.MaxDepth = -1
+	c.DataRateUnit = "bytes"
+	c.BufferSize = SizeSuffix(16 << 20)
+	c.UserAgent = "rclone/" + Version
+	c.StreamingUploadCutoff = SizeSuffix(100 * 1024)
+	c.StatsFileNameLength = 40
+	c.AskPassword = true
+	c.TPSLimitBurst = 1
+	c.MaxTransfer = -1
+	c.MaxBacklog = 10000
+
+	return c
+}
+
+// ConfigToEnv converts an config section and name, eg ("myremote",
+// "ignore-size") into an environment name
+// "RCLONE_CONFIG_MYREMOTE_IGNORE_SIZE"
+func ConfigToEnv(section, name string) string {
+	return "RCLONE_CONFIG_" + strings.ToUpper(strings.Replace(section+"_"+name, "-", "_", -1))
+}
+
+// OptionToEnv converts an option name, eg "ignore-size" into an
+// environment name "RCLONE_IGNORE_SIZE"
+func OptionToEnv(name string) string {
+	return "RCLONE_" + strings.ToUpper(strings.Replace(name, "-", "_", -1))
+}

vendor/github.com/ncw/rclone/fs/config/config_other.go

@@ -0,0 +1,10 @@
+// Read, write and edit the config file
+// Non-unix specific functions.
+
+// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris
+
+package config
+
+// attemptCopyGroups tries to keep the group the same, which only makes sense
+// for system with user-group-world permission model.
+func attemptCopyGroup(fromPath, toPath string) {}

vendor/github.com/ncw/rclone/fs/config/config_read_password.go

@@ -0,0 +1,29 @@
+// ReadPassword for OSes which are supported by golang.org/x/crypto/ssh/terminal
+// See https://github.com/golang/go/issues/14441 - plan9
+//     https://github.com/golang/go/issues/13085 - solaris
+
+// +build !solaris,!plan9
+
+package config
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+// ReadPassword reads a password without echoing it to the terminal.
+func ReadPassword() string {
+	stdin := int(os.Stdin.Fd())
+	if !terminal.IsTerminal(stdin) {
+		return ReadLine()
+	}
+	line, err := terminal.ReadPassword(stdin)
+	_, _ = fmt.Fprintln(os.Stderr)
+	if err != nil {
+		log.Fatalf("Failed to read password: %v", err)
+	}
+	return string(line)
+}

vendor/github.com/ncw/rclone/fs/config/config_read_password_unsupported.go

@@ -0,0 +1,12 @@
+// ReadPassword for OSes which are not supported by golang.org/x/crypto/ssh/terminal
+// See https://github.com/golang/go/issues/14441 - plan9
+//     https://github.com/golang/go/issues/13085 - solaris
+
+// +build solaris plan9
+
+package config
+
+// ReadPassword reads a password with echoing it to the terminal.
+func ReadPassword() string {
+	return ReadLine()
+}

vendor/github.com/ncw/rclone/fs/config/config_unix.go

@@ -0,0 +1,37 @@
+// Read, write and edit the config file
+// Unix specific functions.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd solaris
+
+package config
+
+import (
+	"os"
+	"os/user"
+	"strconv"
+	"syscall"
+
+	"github.com/ncw/rclone/fs"
+)
+
+// attemptCopyGroups tries to keep the group the same. User will be the one
+// who is currently running this process.
+func attemptCopyGroup(fromPath, toPath string) {
+	info, err := os.Stat(fromPath)
+	if err != nil || info.Sys() == nil {
+		return
+	}
+	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+		uid := int(stat.Uid)
+		// prefer self over previous owner of file, because it has a higher chance
+		// of success
+		if user, err := user.Current(); err == nil {
+			if tmpUID, err := strconv.Atoi(user.Uid); err == nil {
+				uid = tmpUID
+			}
+		}
+		if err = os.Chown(toPath, uid, int(stat.Gid)); err != nil {
+			fs.Debugf(nil, "Failed to keep previous owner of config file: %v", err)
+		}
+	}
+}

vendor/github.com/ncw/rclone/fs/config/configmap/configmap.go

@@ -0,0 +1,86 @@
+// Package configmap provides an abstraction for reading and writing config
+package configmap
+
+// Getter provides an interface to get config items
+type Getter interface {
+	// Get should get an item with the key passed in and return
+	// the value. If the item is found then it should return true,
+	// otherwise false.
+	Get(key string) (value string, ok bool)
+}
+
+// Setter provides an interface to set config items
+type Setter interface {
+	// Set should set an item into persistent config store.
+	Set(key, value string)
+}
+
+// Mapper provides an interface to read and write config
+type Mapper interface {
+	Getter
+	Setter
+}
+
+// Map provides a wrapper around multiple Setter and
+// Getter interfaces.
+type Map struct {
+	setters []Setter
+	getters []Getter
+}
+
+// New returns an empty Map
+func New() *Map {
+	return &Map{}
+}
+
+// AddGetter appends a getter onto the end of the getters
+func (c *Map) AddGetter(getter Getter) *Map {
+	c.getters = append(c.getters, getter)
+	return c
+}
+
+// AddGetters appends multiple getters onto the end of the getters
+func (c *Map) AddGetters(getters ...Getter) *Map {
+	c.getters = append(c.getters, getters...)
+	return c
+}
+
+// AddSetter appends a setter onto the end of the setters
+func (c *Map) AddSetter(setter Setter) *Map {
+	c.setters = append(c.setters, setter)
+	return c
+}
+
+// Get gets an item with the key passed in and return the value from
+// the first getter. If the item is found then it returns true,
+// otherwise false.
+func (c *Map) Get(key string) (value string, ok bool) {
+	for _, do := range c.getters {
+		value, ok = do.Get(key)
+		if ok {
+			return value, ok
+		}
+	}
+	return "", false
+}
+
+// Set sets an item into all the stored setters.
+func (c *Map) Set(key, value string) {
+	for _, do := range c.setters {
+		do.Set(key, value)
+	}
+}
+
+// Simple is a simple Mapper for testing
+type Simple map[string]string
+
+// Get the value
+func (c Simple) Get(key string) (value string, ok bool) {
+	value, ok = c[key]
+	return value, ok
+}
+
+// Set the value
+func (c Simple) Set(key, value string) {
+	c[key] = value
+}

vendor/github.com/ncw/rclone/fs/config/configstruct/configstruct.go

@@ -0,0 +1,127 @@
+// Package configstruct parses unstructured maps into structures
+package configstruct
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"github.com/ncw/rclone/fs/config/configmap"
+	"github.com/pkg/errors"
+)
+
+var matchUpper = regexp.MustCompile("([A-Z]+)")
+
+// camelToSnake converts CamelCase to snake_case
+func camelToSnake(in string) string {
+	out := matchUpper.ReplaceAllString(in, "_$1")
+	out = strings.ToLower(out)
+	out = strings.Trim(out, "_")
+	return out
+}
+
+// StringToInterface turns in into an interface{} the same type as def
+func StringToInterface(def interface{}, in string) (newValue interface{}, err error) {
+	typ := reflect.TypeOf(def)
+	switch typ.Kind() {
+	case reflect.String:
+		// Pass strings unmodified
+		return in, nil
+	}
+	// Otherwise parse with Sscanln
+	//
+	// This means any types we use here must implement fmt.Scanner
+	o := reflect.New(typ)
+	n, err := fmt.Sscanln(in, o.Interface())
+	if err != nil {
+		return newValue, errors.Wrapf(err, "parsing %q as %T failed", in, def)
+	}
+	if n != 1 {
+		return newValue, errors.New("no items parsed")
+	}
+	return o.Elem().Interface(), nil
+}
+
+// Item descripts a single entry in the options structure
+type Item struct {
+	Name  string // snake_case
+	Field string // CamelCase
+	Num   int    // number of the field in the struct
+	Value interface{}
+}
+
+// Items parses the opt struct and returns a slice of Item objects.
+//
+// opt must be a pointer to a struct.  The struct should have entirely
+// public fields.
+//
+// The config_name is looked up in a struct tag called "config" or if
+// not found is the field name converted from CamelCase to snake_case.
+func Items(opt interface{}) (items []Item, err error) {
+	def := reflect.ValueOf(opt)
+	if def.Kind() != reflect.Ptr {
+		return nil, errors.New("argument must be a pointer")
+	}
+	def = def.Elem() // indirect the pointer
+	if def.Kind() != reflect.Struct {
+		return nil, errors.New("argument must be a pointer to a struct")
+	}
+	defType := def.Type()
+	for i := 0; i < def.NumField(); i++ {
+		field := defType.Field(i)
+		fieldName := field.Name
+		configName, ok := field.Tag.Lookup("config")
+		if !ok {
+			configName = camelToSnake(fieldName)
+		}
+		defaultItem := Item{
+			Name:  configName,
+			Field: fieldName,
+			Num:   i,
+			Value: def.Field(i).Interface(),
+		}
+		items = append(items, defaultItem)
+	}
+	return items, nil
+}
+
+// Set interprets the field names in defaults and looks up config
+// values in the config passed in.  Any values found in config will be
+// set in the opt structure.
+//
+// opt must be a pointer to a struct.  The struct should have entirely
+// public fields.  The field names are converted from CamelCase to
+// snake_case and looked up in the config supplied or a
+// `config:"field_name"` is looked up.
+//
+// If items are found then they are converted from string to native
+// types and set in opt.
+//
+// All the field types in the struct must implement fmt.Scanner.
+func Set(config configmap.Getter, opt interface{}) (err error) {
+	defaultItems, err := Items(opt)
+	if err != nil {
+		return err
+	}
+	defStruct := reflect.ValueOf(opt).Elem()
+	for _, defaultItem := range defaultItems {
+		newValue := defaultItem.Value
+		if configValue, ok := config.Get(defaultItem.Name); ok {
+			var newNewValue interface{}
+			newNewValue, err = StringToInterface(newValue, configValue)
+			if err != nil {
+				// Mask errors if setting an empty string as
+				// it isn't valid for all types.  This makes
+				// empty string be the equivalent of unset.
+				if configValue != "" {
+					return errors.Wrapf(err, "couldn't parse config item %q = %q as %T", defaultItem.Name, configValue, defaultItem.Value)
+				}
+			} else {
+				newValue = newNewValue
+			}
+		}
+		defStruct.Field(defaultItem.Num).Set(reflect.ValueOf(newValue))
+	}
+	return nil
+}

vendor/github.com/ncw/rclone/fs/config/obscure/obscure.go

@@ -0,0 +1,94 @@
+// Package obscure contains the Obscure and Reveal commands
+package obscure
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/base64"
+	"io"
+	"log"
+
+	"github.com/pkg/errors"
+)
+
+// crypt internals
+var (
+	cryptKey = []byte{
+		0x9c, 0x93, 0x5b, 0x48, 0x73, 0x0a, 0x55, 0x4d,
+		0x6b, 0xfd, 0x7c, 0x63, 0xc8, 0x86, 0xa9, 0x2b,
+		0xd3, 0x90, 0x19, 0x8e, 0xb8, 0x12, 0x8a, 0xfb,
+		0xf4, 0xde, 0x16, 0x2b, 0x8b, 0x95, 0xf6, 0x38,
+	}
+	cryptBlock cipher.Block
+	cryptRand  = rand.Reader
+)
+
+// crypt transforms in to out using iv under AES-CTR.
+//
+// in and out may be the same buffer.
+//
+// Note encryption and decryption are the same operation
+func crypt(out, in, iv []byte) error {
+	if cryptBlock == nil {
+		var err error
+		cryptBlock, err = aes.NewCipher(cryptKey)
+		if err != nil {
+			return err
+		}
+	}
+	stream := cipher.NewCTR(cryptBlock, iv)
+	stream.XORKeyStream(out, in)
+	return nil
+}
+
+// Obscure a value
+//
+// This is done by encrypting with AES-CTR
+func Obscure(x string) (string, error) {
+	plaintext := []byte(x)
+	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
+	iv := ciphertext[:aes.BlockSize]
+	if _, err := io.ReadFull(cryptRand, iv); err != nil {
+		return "", errors.Wrap(err, "failed to read iv")
+	}
+	if err := crypt(ciphertext[aes.BlockSize:], plaintext, iv); err != nil {
+		return "", errors.Wrap(err, "encrypt failed")
+	}
+	return base64.RawURLEncoding.EncodeToString(ciphertext), nil
+}
+
+// MustObscure obscures a value, exiting with a fatal error if it failed
+func MustObscure(x string) string {
+	out, err := Obscure(x)
+	if err != nil {
+		log.Fatalf("Obscure failed: %v", err)
+	}
+	return out
+}
+
+// Reveal an obscured value
+func Reveal(x string) (string, error) {
+	ciphertext, err := base64.RawURLEncoding.DecodeString(x)
+	if err != nil {
+		return "", errors.Wrap(err, "base64 decode failed when revealing password - is it obscured?")
+	}
+	if len(ciphertext) < aes.BlockSize {
+		return "", errors.New("input too short when revealing password - is it obscured?")
+	}
+	buf := ciphertext[aes.BlockSize:]
+	iv := ciphertext[:aes.BlockSize]
+	if err := crypt(buf, buf, iv); err != nil {
+		return "", errors.Wrap(err, "decrypt failed when revealing password - is it obscured?")
+	}
+	return string(buf), nil
+}
+
+// MustReveal reveals an obscured value, exiting with a fatal error if it failed
+func MustReveal(x string) string {
+	out, err := Reveal(x)
+	if err != nil {
+		log.Fatalf("Reveal failed: %v", err)
+	}
+	return out
+}

vendor/github.com/ncw/rclone/fs/config_list.go

@@ -0,0 +1,94 @@
+package fs
+
+import (
+	"bytes"
+	"encoding/csv"
+	"fmt"
+)
+
+// CommaSepList is a comma separated config value
+// It uses the encoding/csv rules for quoting and escaping
+type CommaSepList []string
+
+// SpaceSepList is a space separated config value
+// It uses the encoding/csv rules for quoting and escaping
+type SpaceSepList []string
+
+type genericList []string
+
+func (l CommaSepList) String() string {
+	return genericList(l).string(',')
+}
+
+// Set the List entries
+func (l *CommaSepList) Set(s string) error {
+	return (*genericList)(l).set(',', []byte(s))
+}
+
+// Type of the value
+func (CommaSepList) Type() string {
+	return "[]string"
+}
+
+// Scan implements the fmt.Scanner interface
+func (l *CommaSepList) Scan(s fmt.ScanState, ch rune) error {
+	return (*genericList)(l).scan(',', s, ch)
+}
+
+func (l SpaceSepList) String() string {
+	return genericList(l).string(' ')
+}
+
+// Set the List entries
+func (l *SpaceSepList) Set(s string) error {
+	return (*genericList)(l).set(' ', []byte(s))
+}
+
+// Type of the value
+func (SpaceSepList) Type() string {
+	return "[]string"
+}
+
+// Scan implements the fmt.Scanner interface
+func (l *SpaceSepList) Scan(s fmt.ScanState, ch rune) error {
+	return (*genericList)(l).scan(' ', s, ch)
+}
+
+func (gl genericList) string(sep rune) string {
+	var buf bytes.Buffer
+	w := csv.NewWriter(&buf)
+	w.Comma = sep
+	err := w.Write(gl)
+	if err != nil {
+		// can only happen if w.Comma is invalid
+		panic(err)
+	}
+	w.Flush()
+	return string(bytes.TrimSpace(buf.Bytes()))
+}
+
+func (gl *genericList) set(sep rune, b []byte) error {
+	if len(b) == 0 {
+		*gl = nil
+		return nil
+	}
+	r := csv.NewReader(bytes.NewReader(b))
+	r.Comma = sep
+
+	record, err := r.Read()
+	switch _err := err.(type) {
+	case nil:
+		*gl = record
+	case *csv.ParseError:
+		err = _err.Err // remove line numbers from the error message
+	}
+	return err
+}
+
+func (gl *genericList) scan(sep rune, s fmt.ScanState, ch rune) error {
+	token, err := s.Token(true, func(rune) bool { return true })
+	if err != nil {
+		return err
+	}
+	return gl.set(sep, bytes.TrimSpace(token))
+}

vendor/github.com/ncw/rclone/fs/deletemode.go

@@ -0,0 +1,14 @@
+package fs
+
+// DeleteMode describes the possible delete modes in the config
+type DeleteMode byte
+
+// DeleteMode constants
+const (
+	DeleteModeOff DeleteMode = iota
+	DeleteModeBefore
+	DeleteModeDuring
+	DeleteModeAfter
+	DeleteModeOnly
+	DeleteModeDefault = DeleteModeAfter
+)

vendor/github.com/ncw/rclone/fs/dir.go

@@ -0,0 +1,97 @@
+package fs
+
+import "time"
+
+// Dir describes an unspecialized directory for directory/container/bucket lists
+type Dir struct {
+	remote  string    // name of the directory
+	modTime time.Time // modification or creation time - IsZero for unknown
+	size    int64     // size of directory and contents or -1 if unknown
+	items   int64     // number of objects or -1 for unknown
+	id      string    // optional ID
+}
+
+// NewDir creates an unspecialized Directory object
+func NewDir(remote string, modTime time.Time) *Dir {
+	return &Dir{
+		remote:  remote,
+		modTime: modTime,
+		size:    -1,
+		items:   -1,
+	}
+}
+
+// NewDirCopy creates an unspecialized copy of the Directory object passed in
+func NewDirCopy(d Directory) *Dir {
+	return &Dir{
+		remote:  d.Remote(),
+		modTime: d.ModTime(),
+		size:    d.Size(),
+		items:   d.Items(),
+	}
+}
+
+// String returns the name
+func (d *Dir) String() string {
+	return d.remote
+}
+
+// Remote returns the remote path
+func (d *Dir) Remote() string {
+	return d.remote
+}
+
+// SetRemote sets the remote
+func (d *Dir) SetRemote(remote string) *Dir {
+	d.remote = remote
+	return d
+}
+
+// ID gets the optional ID
+func (d *Dir) ID() string {
+	return d.id
+}
+
+// SetID sets the optional ID
+func (d *Dir) SetID(id string) *Dir {
+	d.id = id
+	return d
+}
+
+// ModTime returns the modification date of the file
+// It should return a best guess if one isn't available
+func (d *Dir) ModTime() time.Time {
+	if !d.modTime.IsZero() {
+		return d.modTime
+	}
+	return time.Now()
+}
+
+// Size returns the size of the file
+func (d *Dir) Size() int64 {
+	return d.size
+}
+
+// SetSize sets the size of the directory
+func (d *Dir) SetSize(size int64) *Dir {
+	d.size = size
+	return d
+}
+
+// Items returns the count of items in this directory or this
+// directory and subdirectories if known, -1 for unknown
+func (d *Dir) Items() int64 {
+	return d.items
+}
+
+// SetItems sets the number of items in the directory
+func (d *Dir) SetItems(items int64) *Dir {
+	d.items = items
+	return d
+}
+
+// Check interfaces
+var (
+	_ DirEntry  = (*Dir)(nil)
+	_ Directory = (*Dir)(nil)
+)

vendor/github.com/ncw/rclone/fs/direntries.go

@@ -0,0 +1,81 @@
+package fs
+
+import "fmt"
+
+// DirEntries is a slice of Object or *Dir
+type DirEntries []DirEntry
+
+// Len is part of sort.Interface.
+func (ds DirEntries) Len() int {
+	return len(ds)
+}
+
+// Swap is part of sort.Interface.
+func (ds DirEntries) Swap(i, j int) {
+	ds[i], ds[j] = ds[j], ds[i]
+}
+
+// Less is part of sort.Interface.
+func (ds DirEntries) Less(i, j int) bool {
+	return ds[i].Remote() < ds[j].Remote()
+}
+
+// ForObject runs the function supplied on every object in the entries
+func (ds DirEntries) ForObject(fn func(o Object)) {
+	for _, entry := range ds {
+		o, ok := entry.(Object)
+		if ok {
+			fn(o)
+		}
+	}
+}
+
+// ForObjectError runs the function supplied on every object in the entries
+func (ds DirEntries) ForObjectError(fn func(o Object) error) error {
+	for _, entry := range ds {
+		o, ok := entry.(Object)
+		if ok {
+			err := fn(o)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// ForDir runs the function supplied on every Directory in the entries
+func (ds DirEntries) ForDir(fn func(dir Directory)) {
+	for _, entry := range ds {
+		dir, ok := entry.(Directory)
+		if ok {
+			fn(dir)
+		}
+	}
+}
+
+// ForDirError runs the function supplied on every Directory in the entries
+func (ds DirEntries) ForDirError(fn func(dir Directory) error) error {
+	for _, entry := range ds {
+		dir, ok := entry.(Directory)
+		if ok {
+			err := fn(dir)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// DirEntryType returns a string description of the DirEntry, either
+// "object", "directory" or "unknown type XXX"
+func DirEntryType(d DirEntry) string {
+	switch d.(type) {
+	case Object:
+		return "object"
+	case Directory:
+		return "directory"
+	}
+	return fmt.Sprintf("unknown type %T", d)
+}

vendor/github.com/ncw/rclone/fs/driveletter/driveletter.go

@@ -0,0 +1,14 @@
+// Package driveletter returns whether a name is a valid drive letter
+
+// +build !windows
+
+package driveletter
+
+// IsDriveLetter returns a bool indicating whether name is a valid
+// Windows drive letter
+//
+// On non windows platforms we don't have drive letters so we always
+// return false
+func IsDriveLetter(name string) bool {
+	return false
+}

vendor/github.com/ncw/rclone/fs/driveletter/driveletter_windows.go

@@ -0,0 +1,13 @@
+// +build windows
+
+package driveletter
+
+// IsDriveLetter returns a bool indicating whether name is a valid
+// Windows drive letter
+func IsDriveLetter(name string) bool {
+	if len(name) != 1 {
+		return false
+	}
+	c := name[0]
+	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+}

vendor/github.com/ncw/rclone/fs/dump.go

@@ -0,0 +1,93 @@
+package fs
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// DumpFlags describes the Dump options in force
+type DumpFlags int
+
+// DumpFlags definitions
+const (
+	DumpHeaders DumpFlags = 1 << iota
+	DumpBodies
+	DumpRequests
+	DumpResponses
+	DumpAuth
+	DumpFilters
+	DumpGoRoutines
+	DumpOpenFiles
+)
+
+var dumpFlags = []struct {
+	flag DumpFlags
+	name string
+}{
+	{DumpHeaders, "headers"},
+	{DumpBodies, "bodies"},
+	{DumpRequests, "requests"},
+	{DumpResponses, "responses"},
+	{DumpAuth, "auth"},
+	{DumpFilters, "filters"},
+	{DumpGoRoutines, "goroutines"},
+	{DumpOpenFiles, "openfiles"},
+}
+
+// DumpFlagsList is a list of dump flags used in the help
+var DumpFlagsList string
+
+func init() {
+	// calculate the dump flags list
+	var out []string
+	for _, info := range dumpFlags {
+		out = append(out, info.name)
+	}
+	DumpFlagsList = strings.Join(out, ",")
+}
+
+// String turns a DumpFlags into a string
+func (f DumpFlags) String() string {
+	var out []string
+	for _, info := range dumpFlags {
+		if f&info.flag != 0 {
+			out = append(out, info.name)
+			f &^= info.flag
+		}
+	}
+	if f != 0 {
+		out = append(out, fmt.Sprintf("Unknown-0x%X", int(f)))
+	}
+	return strings.Join(out, ",")
+}
+
+// Set a DumpFlags as a comma separated list of flags
+func (f *DumpFlags) Set(s string) error {
+	var flags DumpFlags
+	parts := strings.Split(s, ",")
+	for _, part := range parts {
+		found := false
+		part = strings.ToLower(strings.TrimSpace(part))
+		if part == "" {
+			continue
+		}
+		for _, info := range dumpFlags {
+			if part == info.name {
+				found = true
+				flags |= info.flag
+			}
+		}
+		if !found {
+			return errors.Errorf("Unknown dump flag %q", part)
+		}
+	}
+	*f = flags
+	return nil
+}
+
+// Type of the value
+func (f *DumpFlags) Type() string {
+	return "string"
+}

vendor/github.com/ncw/rclone/fs/filter/filter.go

@@ -0,0 +1,498 @@
+// Package filter controls the filtering of files
+package filter
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/pkg/errors"
+)
+
+// Active is the globally active filter
+var Active = mustNewFilter(nil)
+
+// rule is one filter rule
+type rule struct {
+	Include bool
+	Regexp  *regexp.Regexp
+}
+
+// Match returns true if rule matches path
+func (r *rule) Match(path string) bool {
+	return r.Regexp.MatchString(path)
+}
+
+// String the rule
+func (r *rule) String() string {
+	c := "-"
+	if r.Include {
+		c = "+"
+	}
+	return fmt.Sprintf("%s %s", c, r.Regexp.String())
+}
+
+// rules is a slice of rules
+type rules struct {
+	rules    []rule
+	existing map[string]struct{}
+}
+
+// add adds a rule if it doesn't exist already
+func (rs *rules) add(Include bool, re *regexp.Regexp) {
+	if rs.existing == nil {
+		rs.existing = make(map[string]struct{})
+	}
+	newRule := rule{
+		Include: Include,
+		Regexp:  re,
+	}
+	newRuleString := newRule.String()
+	if _, ok := rs.existing[newRuleString]; ok {
+		return // rule already exists
+	}
+	rs.rules = append(rs.rules, newRule)
+	rs.existing[newRuleString] = struct{}{}
+}
+
+// clear clears all the rules
+func (rs *rules) clear() {
+	rs.rules = nil
+	rs.existing = nil
+}
+
+// len returns the number of rules
+func (rs *rules) len() int {
+	return len(rs.rules)
+}
+
+// FilesMap describes the map of files to transfer
+type FilesMap map[string]struct{}
+
+// Opt configues the filter
+type Opt struct {
+	DeleteExcluded bool
+	FilterRule     []string
+	FilterFrom     []string
+	ExcludeRule    []string
+	ExcludeFrom    []string
+	ExcludeFile    string
+	IncludeRule    []string
+	IncludeFrom    []string
+	FilesFrom      []string
+	MinAge         fs.Duration
+	MaxAge         fs.Duration
+	MinSize        fs.SizeSuffix
+	MaxSize        fs.SizeSuffix
+}
+
+// DefaultOpt is the default config for the filter
+var DefaultOpt = Opt{
+	MinAge:  fs.DurationOff,
+	MaxAge:  fs.DurationOff,
+	MinSize: fs.SizeSuffix(-1),
+	MaxSize: fs.SizeSuffix(-1),
+}
+
+// Filter describes any filtering in operation
+type Filter struct {
+	Opt         Opt
+	ModTimeFrom time.Time
+	ModTimeTo   time.Time
+	fileRules   rules
+	dirRules    rules
+	files       FilesMap // files if filesFrom
+	dirs        FilesMap // dirs from filesFrom
+}
+
+// NewFilter parses the command line options and creates a Filter
+// object.  If opt is nil, then DefaultOpt will be used
+func NewFilter(opt *Opt) (f *Filter, err error) {
+	f = &Filter{}
+
+	// Make a copy of the options
+	if opt != nil {
+		f.Opt = *opt
+	} else {
+		f.Opt = DefaultOpt
+	}
+
+	// Filter flags
+	if f.Opt.MinAge.IsSet() {
+		f.ModTimeTo = time.Now().Add(-time.Duration(f.Opt.MinAge))
+		fs.Debugf(nil, "--min-age %v to %v", f.Opt.MinAge, f.ModTimeTo)
+	}
+	if f.Opt.MaxAge.IsSet() {
+		f.ModTimeFrom = time.Now().Add(-time.Duration(f.Opt.MaxAge))
+		if !f.ModTimeTo.IsZero() && f.ModTimeTo.Before(f.ModTimeFrom) {
+			log.Fatal("filter: --min-age can't be larger than --max-age")
+		}
+		fs.Debugf(nil, "--max-age %v to %v", f.Opt.MaxAge, f.ModTimeFrom)
+	}
+
+	addImplicitExclude := false
+	foundExcludeRule := false
+
+	for _, rule := range f.Opt.IncludeRule {
+		err = f.Add(true, rule)
+		if err != nil {
+			return nil, err
+		}
+		addImplicitExclude = true
+	}
+	for _, rule := range f.Opt.IncludeFrom {
+		err := forEachLine(rule, func(line string) error {
+			return f.Add(true, line)
+		})
+		if err != nil {
+			return nil, err
+		}
+		addImplicitExclude = true
+	}
+	for _, rule := range f.Opt.ExcludeRule {
+		err = f.Add(false, rule)
+		if err != nil {
+			return nil, err
+		}
+		foundExcludeRule = true
+	}
+	for _, rule := range f.Opt.ExcludeFrom {
+		err := forEachLine(rule, func(line string) error {
+			return f.Add(false, line)
+		})
+		if err != nil {
+			return nil, err
+		}
+		foundExcludeRule = true
+	}
+
+	if addImplicitExclude && foundExcludeRule {
+		fs.Errorf(nil, "Using --filter is recommended instead of both --include and --exclude as the order they are parsed in is indeterminate")
+	}
+
+	for _, rule := range f.Opt.FilterRule {
+		err = f.AddRule(rule)
+		if err != nil {
+			return nil, err
+		}
+	}
+	for _, rule := range f.Opt.FilterFrom {
+		err := forEachLine(rule, f.AddRule)
+		if err != nil {
+			return nil, err
+		}
+	}
+	for _, rule := range f.Opt.FilesFrom {
+		f.initAddFile() // init to show --files-from set even if no files within
+		err := forEachLine(rule, func(line string) error {
+			return f.AddFile(line)
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+	if addImplicitExclude {
+		err = f.Add(false, "/**")
+		if err != nil {
+			return nil, err
+		}
+	}
+	if fs.Config.Dump&fs.DumpFilters != 0 {
+		fmt.Println("--- start filters ---")
+		fmt.Println(f.DumpFilters())
+		fmt.Println("--- end filters ---")
+	}
+	return f, nil
+}
+
+func mustNewFilter(opt *Opt) *Filter {
+	f, err := NewFilter(opt)
+	if err != nil {
+		panic(err)
+	}
+	return f
+}
+
+// addDirGlobs adds directory globs from the file glob passed in
+func (f *Filter) addDirGlobs(Include bool, glob string) error {
+	for _, dirGlob := range globToDirGlobs(glob) {
+		// Don't add "/" as we always include the root
+		if dirGlob == "/" {
+			continue
+		}
+		dirRe, err := globToRegexp(dirGlob)
+		if err != nil {
+			return err
+		}
+		f.dirRules.add(Include, dirRe)
+	}
+	return nil
+}
+
+// Add adds a filter rule with include or exclude status indicated
+func (f *Filter) Add(Include bool, glob string) error {
+	isDirRule := strings.HasSuffix(glob, "/")
+	isFileRule := !isDirRule
+	if strings.Contains(glob, "**") {
+		isDirRule, isFileRule = true, true
+	}
+	re, err := globToRegexp(glob)
+	if err != nil {
+		return err
+	}
+	if isFileRule {
+		f.fileRules.add(Include, re)
+		// If include rule work out what directories are needed to scan
+		// if exclude rule, we can't rule anything out
+		// Unless it is `*` which matches everything
+		// NB ** and /** are DirRules
+		if Include || glob == "*" {
+			err = f.addDirGlobs(Include, glob)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	if isDirRule {
+		f.dirRules.add(Include, re)
+	}
+	return nil
+}
+
+// AddRule adds a filter rule with include/exclude indicated by the prefix
+//
+// These are
+//
+//   + glob
+//   - glob
+//   !
+//
+// '+' includes the glob, '-' excludes it and '!' resets the filter list
+//
+// Line comments may be introduced with '#' or ';'
+func (f *Filter) AddRule(rule string) error {
+	switch {
+	case rule == "!":
+		f.Clear()
+		return nil
+	case strings.HasPrefix(rule, "- "):
+		return f.Add(false, rule[2:])
+	case strings.HasPrefix(rule, "+ "):
+		return f.Add(true, rule[2:])
+	}
+	return errors.Errorf("malformed rule %q", rule)
+}
+
+// initAddFile creates f.files and f.dirs
+func (f *Filter) initAddFile() {
+	if f.files == nil {
+		f.files = make(FilesMap)
+		f.dirs = make(FilesMap)
+	}
+}
+
+// AddFile adds a single file to the files from list
+func (f *Filter) AddFile(file string) error {
+	f.initAddFile()
+	file = strings.Trim(file, "/")
+	f.files[file] = struct{}{}
+	// Put all the parent directories into f.dirs
+	for {
+		file = path.Dir(file)
+		if file == "." {
+			break
+		}
+		if _, found := f.dirs[file]; found {
+			break
+		}
+		f.dirs[file] = struct{}{}
+	}
+	return nil
+}
+
+// Files returns all the files from the `--files-from` list
+//
+// It may be nil if the list is empty
+func (f *Filter) Files() FilesMap {
+	return f.files
+}
+
+// Clear clears all the filter rules
+func (f *Filter) Clear() {
+	f.fileRules.clear()
+	f.dirRules.clear()
+}
+
+// InActive returns false if any filters are active
+func (f *Filter) InActive() bool {
+	return (f.files == nil &&
+		f.ModTimeFrom.IsZero() &&
+		f.ModTimeTo.IsZero() &&
+		f.Opt.MinSize < 0 &&
+		f.Opt.MaxSize < 0 &&
+		f.fileRules.len() == 0 &&
+		f.dirRules.len() == 0 &&
+		len(f.Opt.ExcludeFile) == 0)
+}
+
+// includeRemote returns whether this remote passes the filter rules.
+func (f *Filter) includeRemote(remote string) bool {
+	for _, rule := range f.fileRules.rules {
+		if rule.Match(remote) {
+			return rule.Include
+		}
+	}
+	return true
+}
+
+// ListContainsExcludeFile checks if exclude file is present in the list.
+func (f *Filter) ListContainsExcludeFile(entries fs.DirEntries) bool {
+	if len(f.Opt.ExcludeFile) == 0 {
+		return false
+	}
+	for _, entry := range entries {
+		obj, ok := entry.(fs.Object)
+		if ok {
+			basename := path.Base(obj.Remote())
+			if basename == f.Opt.ExcludeFile {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// IncludeDirectory returns a function which checks whether this
+// directory should be included in the sync or not.
+func (f *Filter) IncludeDirectory(fs fs.Fs) func(string) (bool, error) {
+	return func(remote string) (bool, error) {
+		remote = strings.Trim(remote, "/")
+		// first check if we need to remove directory based on
+		// the exclude file
+		excl, err := f.DirContainsExcludeFile(fs, remote)
+		if err != nil {
+			return false, err
+		}
+		if excl {
+			return false, nil
+		}
+
+		// filesFrom takes precedence
+		if f.files != nil {
+			_, include := f.dirs[remote]
+			return include, nil
+		}
+		remote += "/"
+		for _, rule := range f.dirRules.rules {
+			if rule.Match(remote) {
+				return rule.Include, nil
+			}
+		}
+
+		return true, nil
+	}
+}
+
+// DirContainsExcludeFile checks if exclude file is present in a
+// directroy. If fs is nil, it works properly if ExcludeFile is an
+// empty string (for testing).
+func (f *Filter) DirContainsExcludeFile(fremote fs.Fs, remote string) (bool, error) {
+	if len(f.Opt.ExcludeFile) > 0 {
+		exists, err := fs.FileExists(fremote, path.Join(remote, f.Opt.ExcludeFile))
+		if err != nil {
+			return false, err
+		}
+		if exists {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// Include returns whether this object should be included into the
+// sync or not
+func (f *Filter) Include(remote string, size int64, modTime time.Time) bool {
+	// filesFrom takes precedence
+	if f.files != nil {
+		_, include := f.files[remote]
+		return include
+	}
+	if !f.ModTimeFrom.IsZero() && modTime.Before(f.ModTimeFrom) {
+		return false
+	}
+	if !f.ModTimeTo.IsZero() && modTime.After(f.ModTimeTo) {
+		return false
+	}
+	if f.Opt.MinSize >= 0 && size < int64(f.Opt.MinSize) {
+		return false
+	}
+	if f.Opt.MaxSize >= 0 && size > int64(f.Opt.MaxSize) {
+		return false
+	}
+	return f.includeRemote(remote)
+}
+
+// IncludeObject returns whether this object should be included into
+// the sync or not. This is a convenience function to avoid calling
+// o.ModTime(), which is an expensive operation.
+func (f *Filter) IncludeObject(o fs.Object) bool {
+	var modTime time.Time
+
+	if !f.ModTimeFrom.IsZero() || !f.ModTimeTo.IsZero() {
+		modTime = o.ModTime()
+	} else {
+		modTime = time.Unix(0, 0)
+	}
+
+	return f.Include(o.Remote(), o.Size(), modTime)
+}
+
+// forEachLine calls fn on every line in the file pointed to by path
+//
+// It ignores empty lines and lines starting with '#' or ';'
+func forEachLine(path string, fn func(string) error) (err error) {
+	in, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer fs.CheckClose(in, &err)
+	scanner := bufio.NewScanner(in)
+	for scanner.Scan() {
+		line := scanner.Text()
+		line = strings.TrimSpace(line)
+		if len(line) == 0 || line[0] == '#' || line[0] == ';' {
+			continue
+		}
+		err := fn(line)
+		if err != nil {
+			return err
+		}
+	}
+	return scanner.Err()
+}
+
+// DumpFilters dumps the filters in textual form, 1 per line
+func (f *Filter) DumpFilters() string {
+	rules := []string{}
+	if !f.ModTimeFrom.IsZero() {
+		rules = append(rules, fmt.Sprintf("Last-modified date must be equal or greater than: %s", f.ModTimeFrom.String()))
+	}
+	if !f.ModTimeTo.IsZero() {
+		rules = append(rules, fmt.Sprintf("Last-modified date must be equal or less than: %s", f.ModTimeTo.String()))
+	}
+	rules = append(rules, "--- File filter rules ---")
+	for _, rule := range f.fileRules.rules {
+		rules = append(rules, rule.String())
+	}
+	rules = append(rules, "--- Directory filter rules ---")
+	for _, dirRule := range f.dirRules.rules {
+		rules = append(rules, dirRule.String())
+	}
+	return strings.Join(rules, "\n")
+}

vendor/github.com/ncw/rclone/fs/filter/glob.go

@@ -0,0 +1,166 @@
+// rsync style glob parser
+
+package filter
+
+import (
+	"bytes"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// globToRegexp converts an rsync style glob to a regexp
+//
+// documented in filtering.md
+func globToRegexp(glob string) (*regexp.Regexp, error) {
+	var re bytes.Buffer
+	if strings.HasPrefix(glob, "/") {
+		glob = glob[1:]
+		_, _ = re.WriteRune('^')
+	} else {
+		_, _ = re.WriteString("(^|/)")
+	}
+	consecutiveStars := 0
+	insertStars := func() error {
+		if consecutiveStars > 0 {
+			switch consecutiveStars {
+			case 1:
+				_, _ = re.WriteString(`[^/]*`)
+			case 2:
+				_, _ = re.WriteString(`.*`)
+			default:
+				return errors.Errorf("too many stars in %q", glob)
+			}
+		}
+		consecutiveStars = 0
+		return nil
+	}
+	inBraces := false
+	inBrackets := 0
+	slashed := false
+	for _, c := range glob {
+		if slashed {
+			_, _ = re.WriteRune(c)
+			slashed = false
+			continue
+		}
+		if c != '*' {
+			err := insertStars()
+			if err != nil {
+				return nil, err
+			}
+		}
+		if inBrackets > 0 {
+			_, _ = re.WriteRune(c)
+			if c == '[' {
+				inBrackets++
+			}
+			if c == ']' {
+				inBrackets--
+			}
+			continue
+		}
+		switch c {
+		case '\\':
+			_, _ = re.WriteRune(c)
+			slashed = true
+		case '*':
+			consecutiveStars++
+		case '?':
+			_, _ = re.WriteString(`[^/]`)
+		case '[':
+			_, _ = re.WriteRune(c)
+			inBrackets++
+		case ']':
+			return nil, errors.Errorf("mismatched ']' in glob %q", glob)
+		case '{':
+			if inBraces {
+				return nil, errors.Errorf("can't nest '{' '}' in glob %q", glob)
+			}
+			inBraces = true
+			_, _ = re.WriteRune('(')
+		case '}':
+			if !inBraces {
+				return nil, errors.Errorf("mismatched '{' and '}' in glob %q", glob)
+			}
+			_, _ = re.WriteRune(')')
+			inBraces = false
+		case ',':
+			if inBraces {
+				_, _ = re.WriteRune('|')
+			} else {
+				_, _ = re.WriteRune(c)
+			}
+		case '.', '+', '(', ')', '|', '^', '$': // regexp meta characters not dealt with above
+			_, _ = re.WriteRune('\\')
+			_, _ = re.WriteRune(c)
+		default:
+			_, _ = re.WriteRune(c)
+		}
+	}
+	err := insertStars()
+	if err != nil {
+		return nil, err
+	}
+	if inBrackets > 0 {
+		return nil, errors.Errorf("mismatched '[' and ']' in glob %q", glob)
+	}
+	if inBraces {
+		return nil, errors.Errorf("mismatched '{' and '}' in glob %q", glob)
+	}
+	_, _ = re.WriteRune('$')
+	result, err := regexp.Compile(re.String())
+	if err != nil {
+		return nil, errors.Wrapf(err, "bad glob pattern %q (regexp %q)", glob, re.String())
+	}
+	return result, nil
+}
+
+var (
+	// Can't deal with / or ** in {}
+	tooHardRe = regexp.MustCompile(`{[^{}]*(\*\*|/)[^{}]*}`)
+
+	// Squash all /
+	squashSlash = regexp.MustCompile(`/{2,}`)
+)
+
+// globToDirGlobs takes a file glob and turns it into a series of
+// directory globs.  When matched with a directory (with a trailing /)
+// this should answer the question as to whether this glob could be in
+// this directory.
+func globToDirGlobs(glob string) (out []string) {
+	if tooHardRe.MatchString(glob) {
+		// Can't figure this one out so return any directory might match
+		out = append(out, "/**")
+		return out
+	}
+
+	// Get rid of multiple /s
+	glob = squashSlash.ReplaceAllString(glob, "/")
+
+	// Split on / or **
+	// (** can contain /)
+	for {
+		i := strings.LastIndex(glob, "/")
+		j := strings.LastIndex(glob, "**")
+		what := ""
+		if j > i {
+			i = j
+			what = "**"
+		}
+		if i < 0 {
+			if len(out) == 0 {
+				out = append(out, "/**")
+			}
+			break
+		}
+		glob = glob[:i]
+		newGlob := glob + what + "/"
+		if len(out) == 0 || out[len(out)-1] != newGlob {
+			out = append(out, newGlob)
+		}
+	}
+
+	return out
+}

vendor/github.com/ncw/rclone/fs/fserrors/error.go

@@ -0,0 +1,292 @@
+// Package fserrors provides errors and error handling
+package fserrors
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"reflect"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// Retrier is an optional interface for error as to whether the
+// operation should be retried at a high level.
+//
+// This should be returned from Update or Put methods as required
+type Retrier interface {
+	error
+	Retry() bool
+}
+
+// retryError is a type of error
+type retryError string
+
+// Error interface
+func (r retryError) Error() string {
+	return string(r)
+}
+
+// Retry interface
+func (r retryError) Retry() bool {
+	return true
+}
+
+// Check interface
+var _ Retrier = retryError("")
+
+// RetryErrorf makes an error which indicates it would like to be retried
+func RetryErrorf(format string, a ...interface{}) error {
+	return retryError(fmt.Sprintf(format, a...))
+}
+
+// wrappedRetryError is an error wrapped so it will satisfy the
+// Retrier interface and return true
+type wrappedRetryError struct {
+	error
+}
+
+// Retry interface
+func (err wrappedRetryError) Retry() bool {
+	return true
+}
+
+// Check interface
+var _ Retrier = wrappedRetryError{error(nil)}
+
+// RetryError makes an error which indicates it would like to be retried
+func RetryError(err error) error {
+	if err == nil {
+		err = errors.New("needs retry")
+	}
+	return wrappedRetryError{err}
+}
+
+// IsRetryError returns true if err conforms to the Retry interface
+// and calling the Retry method returns true.
+func IsRetryError(err error) bool {
+	if err == nil {
+		return false
+	}
+	_, err = Cause(err)
+	if r, ok := err.(Retrier); ok {
+		return r.Retry()
+	}
+	return false
+}
+
+// Fataler is an optional interface for error as to whether the
+// operation should cause the entire operation to finish immediately.
+//
+// This should be returned from Update or Put methods as required
+type Fataler interface {
+	error
+	Fatal() bool
+}
+
+// wrappedFatalError is an error wrapped so it will satisfy the
+// Retrier interface and return true
+type wrappedFatalError struct {
+	error
+}
+
+// Fatal interface
+func (err wrappedFatalError) Fatal() bool {
+	return true
+}
+
+// Check interface
+var _ Fataler = wrappedFatalError{error(nil)}
+
+// FatalError makes an error which indicates it is a fatal error and
+// the sync should stop.
+func FatalError(err error) error {
+	if err == nil {
+		err = errors.New("fatal error")
+	}
+	return wrappedFatalError{err}
+}
+
+// IsFatalError returns true if err conforms to the Fatal interface
+// and calling the Fatal method returns true.
+func IsFatalError(err error) bool {
+	if err == nil {
+		return false
+	}
+	_, err = Cause(err)
+	if r, ok := err.(Fataler); ok {
+		return r.Fatal()
+	}
+	return false
+}
+
+// NoRetrier is an optional interface for error as to whether the
+// operation should not be retried at a high level.
+//
+// If only NoRetry errors are returned in a sync then the sync won't
+// be retried.
+//
+// This should be returned from Update or Put methods as required
+type NoRetrier interface {
+	error
+	NoRetry() bool
+}
+
+// wrappedNoRetryError is an error wrapped so it will satisfy the
+// Retrier interface and return true
+type wrappedNoRetryError struct {
+	error
+}
+
+// NoRetry interface
+func (err wrappedNoRetryError) NoRetry() bool {
+	return true
+}
+
+// Check interface
+var _ NoRetrier = wrappedNoRetryError{error(nil)}
+
+// NoRetryError makes an error which indicates the sync shouldn't be
+// retried.
+func NoRetryError(err error) error {
+	return wrappedNoRetryError{err}
+}
+
+// IsNoRetryError returns true if err conforms to the NoRetry
+// interface and calling the NoRetry method returns true.
+func IsNoRetryError(err error) bool {
+	if err == nil {
+		return false
+	}
+	_, err = Cause(err)
+	if r, ok := err.(NoRetrier); ok {
+		return r.NoRetry()
+	}
+	return false
+}
+
+// Cause is a souped up errors.Cause which can unwrap some standard
+// library errors too.  It returns true if any of the intermediate
+// errors had a Timeout() or Temporary() method which returned true.
+func Cause(cause error) (retriable bool, err error) {
+	err = cause
+	for prev := err; err != nil; prev = err {
+		// Check for net error Timeout()
+		if x, ok := err.(interface {
+			Timeout() bool
+		}); ok && x.Timeout() {
+			retriable = true
+		}
+
+		// Check for net error Temporary()
+		if x, ok := err.(interface {
+			Temporary() bool
+		}); ok && x.Temporary() {
+			retriable = true
+		}
+
+		// Unwrap 1 level if possible
+		err = errors.Cause(err)
+		if err == nil {
+			// errors.Cause can return nil which isn't
+			// desirable so pick the previous error in
+			// this case.
+			err = prev
+		}
+		if err == prev {
+			// Unpack any struct or *struct with a field
+			// of name Err which satisfies the error
+			// interface.  This includes *url.Error,
+			// *net.OpError, *os.SyscallError and many
+			// others in the stdlib
+			errType := reflect.TypeOf(err)
+			errValue := reflect.ValueOf(err)
+			if errValue.IsValid() && errType.Kind() == reflect.Ptr {
+				errType = errType.Elem()
+				errValue = errValue.Elem()
+			}
+			if errValue.IsValid() && errType.Kind() == reflect.Struct {
+				if errField := errValue.FieldByName("Err"); errField.IsValid() {
+					errFieldValue := errField.Interface()
+					if newErr, ok := errFieldValue.(error); ok {
+						err = newErr
+					}
+				}
+			}
+		}
+		if err == prev {
+			break
+		}
+	}
+	return retriable, err
+}
+
+// retriableErrorStrings is a list of phrases which when we find it
+// in an an error, we know it is a networking error which should be
+// retried.
+//
+// This is incredibly ugly - if only errors.Cause worked for all
+// errors and all errors were exported from the stdlib.
+var retriableErrorStrings = []string{
+	"use of closed network connection", // internal/poll/fd.go
+	"unexpected EOF reading trailer",   // net/http/transfer.go
+	"transport connection broken",      // net/http/transport.go
+	"http: ContentLength=",             // net/http/transfer.go
+}
+
+// Errors which indicate networking errors which should be retried
+//
+// These are added to in retriable_errors*.go
+var retriableErrors = []error{
+	io.EOF,
+	io.ErrUnexpectedEOF,
+}
+
+// ShouldRetry looks at an error and tries to work out if retrying the
+// operation that caused it would be a good idea. It returns true if
+// the error implements Timeout() or Temporary() or if the error
+// indicates a premature closing of the connection.
+func ShouldRetry(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Find root cause if available
+	retriable, err := Cause(err)
+	if retriable {
+		return true
+	}
+
+	// Check if it is a retriable error
+	for _, retriableErr := range retriableErrors {
+		if err == retriableErr {
+			return true
+		}
+	}
+
+	// Check error strings (yuch!) too
+	errString := err.Error()
+	for _, phrase := range retriableErrorStrings {
+		if strings.Contains(errString, phrase) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// ShouldRetryHTTP returns a boolean as to whether this resp deserves.
+// It checks to see if the HTTP response code is in the slice
+// retryErrorCodes.
+func ShouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool {
+	if resp == nil {
+		return false
+	}
+	for _, e := range retryErrorCodes {
+		if resp.StatusCode == e {
+			return true
+		}
+	}
+	return false
+}

vendor/github.com/ncw/rclone/fs/fserrors/retriable_errors.go

@@ -0,0 +1,21 @@
+// +build !plan9
+
+package fserrors
+
+import (
+	"syscall"
+)
+
+func init() {
+	retriableErrors = append(retriableErrors,
+		syscall.EPIPE,
+		syscall.ETIMEDOUT,
+		syscall.ECONNREFUSED,
+		syscall.EHOSTDOWN,
+		syscall.EHOSTUNREACH,
+		syscall.ECONNABORTED,
+		syscall.EAGAIN,
+		syscall.EWOULDBLOCK,
+		syscall.ECONNRESET,
+	)
+}

vendor/github.com/ncw/rclone/fs/fserrors/retriable_errors_windows.go

@@ -0,0 +1,31 @@
+// +build windows
+
+package fserrors
+
+import (
+	"syscall"
+)
+
+const (
+	WSAECONNABORTED   syscall.Errno = 10053
+	WSAHOST_NOT_FOUND syscall.Errno = 11001
+	WSATRY_AGAIN      syscall.Errno = 11002
+	WSAENETRESET      syscall.Errno = 10052
+	WSAETIMEDOUT      syscall.Errno = 10060
+)
+
+func init() {
+	// append some lower level errors since the standardized ones
+	// don't seem to happen
+	retriableErrors = append(retriableErrors,
+		syscall.WSAECONNRESET,
+		WSAECONNABORTED,
+		WSAHOST_NOT_FOUND,
+		WSATRY_AGAIN,
+		WSAENETRESET,
+		WSAETIMEDOUT,
+		syscall.ERROR_HANDLE_EOF,
+		syscall.ERROR_NETNAME_DELETED,
+		syscall.ERROR_BROKEN_PIPE,
+	)
+}

vendor/github.com/ncw/rclone/fs/fshttp/http.go

@@ -0,0 +1,312 @@
+// Package fshttp contains the common http parts of the config, Transport and Client
+package fshttp
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"reflect"
+	"sync"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"golang.org/x/time/rate"
+)
+
+const (
+	separatorReq  = ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
+	separatorResp = "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"
+)
+
+var (
+	transport   http.RoundTripper
+	noTransport sync.Once
+	tpsBucket   *rate.Limiter // for limiting number of http transactions per second
+)
+
+// StartHTTPTokenBucket starts the token bucket if necessary
+func StartHTTPTokenBucket() {
+	if fs.Config.TPSLimit > 0 {
+		tpsBurst := fs.Config.TPSLimitBurst
+		if tpsBurst < 1 {
+			tpsBurst = 1
+		}
+		tpsBucket = rate.NewLimiter(rate.Limit(fs.Config.TPSLimit), tpsBurst)
+		fs.Infof(nil, "Starting HTTP transaction limiter: max %g transactions/s with burst %d", fs.Config.TPSLimit, tpsBurst)
+	}
+}
+
+// A net.Conn that sets a deadline for every Read or Write operation
+type timeoutConn struct {
+	net.Conn
+	timeout time.Duration
+}
+
+// create a timeoutConn using the timeout
+func newTimeoutConn(conn net.Conn, timeout time.Duration) (c *timeoutConn, err error) {
+	c = &timeoutConn{
+		Conn:    conn,
+		timeout: timeout,
+	}
+	err = c.nudgeDeadline()
+	return
+}
+
+// Nudge the deadline for an idle timeout on by c.timeout if non-zero
+func (c *timeoutConn) nudgeDeadline() (err error) {
+	if c.timeout == 0 {
+		return nil
+	}
+	when := time.Now().Add(c.timeout)
+	return c.Conn.SetDeadline(when)
+}
+
+// readOrWrite bytes doing idle timeouts
+func (c *timeoutConn) readOrWrite(f func([]byte) (int, error), b []byte) (n int, err error) {
+	n, err = f(b)
+	// Don't nudge if no bytes or an error
+	if n == 0 || err != nil {
+		return
+	}
+	// Nudge the deadline on successful Read or Write
+	err = c.nudgeDeadline()
+	return
+}
+
+// Read bytes doing idle timeouts
+func (c *timeoutConn) Read(b []byte) (n int, err error) {
+	return c.readOrWrite(c.Conn.Read, b)
+}
+
+// Write bytes doing idle timeouts
+func (c *timeoutConn) Write(b []byte) (n int, err error) {
+	return c.readOrWrite(c.Conn.Write, b)
+}
+
+// setDefaults for a from b
+//
+// Copy the public members from b to a.  We can't just use a struct
+// copy as Transport contains a private mutex.
+func setDefaults(a, b interface{}) {
+	pt := reflect.TypeOf(a)
+	t := pt.Elem()
+	va := reflect.ValueOf(a).Elem()
+	vb := reflect.ValueOf(b).Elem()
+	for i := 0; i < t.NumField(); i++ {
+		aField := va.Field(i)
+		// Set a from b if it is public
+		if aField.CanSet() {
+			bField := vb.Field(i)
+			aField.Set(bField)
+		}
+	}
+}
+
+// dial with context and timeouts
+func dialContextTimeout(ctx context.Context, network, address string, ci *fs.ConfigInfo) (net.Conn, error) {
+	dialer := NewDialer(ci)
+	c, err := dialer.DialContext(ctx, network, address)
+	if err != nil {
+		return c, err
+	}
+	return newTimeoutConn(c, ci.Timeout)
+}
+
+// NewTransport returns an http.RoundTripper with the correct timeouts
+func NewTransport(ci *fs.ConfigInfo) http.RoundTripper {
+	noTransport.Do(func() {
+		// Start with a sensible set of defaults then override.
+		// This also means we get new stuff when it gets added to go
+		t := new(http.Transport)
+		setDefaults(t, http.DefaultTransport.(*http.Transport))
+		t.Proxy = http.ProxyFromEnvironment
+		t.MaxIdleConnsPerHost = 2 * (ci.Checkers + ci.Transfers + 1)
+		t.MaxIdleConns = 2 * t.MaxIdleConnsPerHost
+		t.TLSHandshakeTimeout = ci.ConnectTimeout
+		t.ResponseHeaderTimeout = ci.Timeout
+		t.TLSClientConfig = &tls.Config{InsecureSkipVerify: ci.InsecureSkipVerify}
+		t.DisableCompression = ci.NoGzip
+		t.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return dialContextTimeout(ctx, network, addr, ci)
+		}
+		t.IdleConnTimeout = 60 * time.Second
+		t.ExpectContinueTimeout = ci.ConnectTimeout
+		// Wrap that http.Transport in our own transport
+		transport = newTransport(ci, t)
+	})
+	return transport
+}
+
+// NewClient returns an http.Client with the correct timeouts
+func NewClient(ci *fs.ConfigInfo) *http.Client {
+	return &http.Client{
+		Transport: NewTransport(ci),
+	}
+}
+
+// Transport is a our http Transport which wraps an http.Transport
+// * Sets the User Agent
+// * Does logging
+type Transport struct {
+	*http.Transport
+	dump          fs.DumpFlags
+	filterRequest func(req *http.Request)
+	userAgent     string
+}
+
+// newTransport wraps the http.Transport passed in and logs all
+// roundtrips including the body if logBody is set.
+func newTransport(ci *fs.ConfigInfo, transport *http.Transport) *Transport {
+	return &Transport{
+		Transport: transport,
+		dump:      ci.Dump,
+		userAgent: ci.UserAgent,
+	}
+}
+
+// SetRequestFilter sets a filter to be used on each request
+func (t *Transport) SetRequestFilter(f func(req *http.Request)) {
+	t.filterRequest = f
+}
+
+// A mutex to protect this map
+var checkedHostMu sync.RWMutex
+
+// A map of servers we have checked for time
+var checkedHost = make(map[string]struct{}, 1)
+
+// Check the server time is the same as ours, once for each server
+func checkServerTime(req *http.Request, resp *http.Response) {
+	host := req.URL.Host
+	if req.Host != "" {
+		host = req.Host
+	}
+	checkedHostMu.RLock()
+	_, ok := checkedHost[host]
+	checkedHostMu.RUnlock()
+	if ok {
+		return
+	}
+	dateString := resp.Header.Get("Date")
+	if dateString == "" {
+		return
+	}
+	date, err := http.ParseTime(dateString)
+	if err != nil {
+		fs.Debugf(nil, "Couldn't parse Date: from server %s: %q: %v", host, dateString, err)
+		return
+	}
+	dt := time.Since(date)
+	const window = 5 * 60 * time.Second
+	if dt > window || dt < -window {
+		fs.Logf(nil, "Time may be set wrong - time from %q is %v different from this computer", host, dt)
+	}
+	checkedHostMu.Lock()
+	checkedHost[host] = struct{}{}
+	checkedHostMu.Unlock()
+}
+
+// cleanAuth gets rid of one authBuf header within the first 4k
+func cleanAuth(buf, authBuf []byte) []byte {
+	// Find how much buffer to check
+	n := 4096
+	if len(buf) < n {
+		n = len(buf)
+	}
+	// See if there is an Authorization: header
+	i := bytes.Index(buf[:n], authBuf)
+	if i < 0 {
+		return buf
+	}
+	i += len(authBuf)
+	// Overwrite the next 4 chars with 'X'
+	for j := 0; i < len(buf) && j < 4; j++ {
+		if buf[i] == '\n' {
+			break
+		}
+		buf[i] = 'X'
+		i++
+	}
+	// Snip out to the next '\n'
+	j := bytes.IndexByte(buf[i:], '\n')
+	if j < 0 {
+		return buf[:i]
+	}
+	n = copy(buf[i:], buf[i+j:])
+	return buf[:i+n]
+}
+
+var authBufs = [][]byte{
+	[]byte("Authorization: "),
+	[]byte("X-Auth-Token: "),
+}
+
+// cleanAuths gets rid of all the possible Auth headers
+func cleanAuths(buf []byte) []byte {
+	for _, authBuf := range authBufs {
+		buf = cleanAuth(buf, authBuf)
+	}
+	return buf
+}
+
+// RoundTrip implements the RoundTripper interface.
+func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
+	// Get transactions per second token first if limiting
+	if tpsBucket != nil {
+		tbErr := tpsBucket.Wait(req.Context())
+		if tbErr != nil {
+			fs.Errorf(nil, "HTTP token bucket error: %v", err)
+		}
+	}
+	// Force user agent
+	req.Header.Set("User-Agent", t.userAgent)
+	// Filter the request if required
+	if t.filterRequest != nil {
+		t.filterRequest(req)
+	}
+	// Logf request
+	if t.dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpAuth|fs.DumpRequests|fs.DumpResponses) != 0 {
+		buf, _ := httputil.DumpRequestOut(req, t.dump&(fs.DumpBodies|fs.DumpRequests) != 0)
+		if t.dump&fs.DumpAuth == 0 {
+			buf = cleanAuths(buf)
+		}
+		fs.Debugf(nil, "%s", separatorReq)
+		fs.Debugf(nil, "%s (req %p)", "HTTP REQUEST", req)
+		fs.Debugf(nil, "%s", string(buf))
+		fs.Debugf(nil, "%s", separatorReq)
+	}
+	// Do round trip
+	resp, err = t.Transport.RoundTrip(req)
+	// Logf response
+	if t.dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpAuth|fs.DumpRequests|fs.DumpResponses) != 0 {
+		fs.Debugf(nil, "%s", separatorResp)
+		fs.Debugf(nil, "%s (req %p)", "HTTP RESPONSE", req)
+		if err != nil {
+			fs.Debugf(nil, "Error: %v", err)
+		} else {
+			buf, _ := httputil.DumpResponse(resp, t.dump&(fs.DumpBodies|fs.DumpResponses) != 0)
+			fs.Debugf(nil, "%s", string(buf))
+		}
+		fs.Debugf(nil, "%s", separatorResp)
+	}
+	if err == nil {
+		checkServerTime(req, resp)
+	}
+	return resp, err
+}
+
+// NewDialer creates a net.Dialer structure with Timeout, Keepalive
+// and LocalAddr set from rclone flags.
+func NewDialer(ci *fs.ConfigInfo) *net.Dialer {
+	dialer := &net.Dialer{
+		Timeout:   ci.ConnectTimeout,
+		KeepAlive: 30 * time.Second,
+	}
+	if ci.BindAddr != nil {
+		dialer.LocalAddr = &net.TCPAddr{IP: ci.BindAddr}
+	}
+	return dialer
+}

vendor/github.com/ncw/rclone/fs/fspath/path.go

@@ -0,0 +1,50 @@
+// Package fspath contains routines for fspath manipulation
+package fspath
+
+import (
+	"path"
+	"path/filepath"
+	"regexp"
+
+	"github.com/ncw/rclone/fs/driveletter"
+)
+
+// Matcher is a pattern to match an rclone URL
+var Matcher = regexp.MustCompile(`^(:?[\w_ -]+):(.*)$`)
+
+// Parse deconstructs a remote path into configName and fsPath
+//
+// If the path is a local path then configName will be returned as "".
+//
+// So "remote:path/to/dir" will return "remote", "path/to/dir"
+// and "/path/to/local" will return ("", "/path/to/local")
+//
+// Note that this will turn \ into / in the fsPath on Windows
+func Parse(path string) (configName, fsPath string) {
+	parts := Matcher.FindStringSubmatch(path)
+	configName, fsPath = "", path
+	if parts != nil && !driveletter.IsDriveLetter(parts[1]) {
+		configName, fsPath = parts[1], parts[2]
+	}
+	// change native directory separators to / if there are any
+	fsPath = filepath.ToSlash(fsPath)
+	return configName, fsPath
+}
+
+// Split splits a remote into a parent and a leaf
+//
+// if it returns leaf as an empty string then remote is a directory
+//
+// if it returns parent as an empty string then that means the current directory
+//
+// The returned values have the property that parent + leaf == remote
+// (except under Windows where \ will be translated into /)
+func Split(remote string) (parent string, leaf string) {
+	remoteName, remotePath := Parse(remote)
+	if remoteName != "" {
+		remoteName += ":"
+	}
+	// Construct new remote name without last segment
+	parent, leaf = path.Split(remotePath)
+	return remoteName + parent, leaf
+}

vendor/github.com/ncw/rclone/fs/hash/hash.go

@@ -0,0 +1,308 @@
+package hash
+
+import (
+	"crypto/md5"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"io"
+	"strings"
+
+	"github.com/ncw/rclone/backend/dropbox/dbhash"
+	"github.com/ncw/rclone/backend/onedrive/quickxorhash"
+	"github.com/pkg/errors"
+)
+
+// Type indicates a standard hashing algorithm
+type Type int
+
+// ErrUnsupported should be returned by filesystem,
+// if it is requested to deliver an unsupported hash type.
+var ErrUnsupported = errors.New("hash type not supported")
+
+const (
+	// MD5 indicates MD5 support
+	MD5 Type = 1 << iota
+
+	// SHA1 indicates SHA-1 support
+	SHA1
+
+	// Dropbox indicates Dropbox special hash
+	// https://www.dropbox.com/developers/reference/content-hash
+	Dropbox
+
+	// QuickXorHash indicates Microsoft onedrive hash
+	// https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
+	QuickXorHash
+
+	// None indicates no hashes are supported
+	None Type = 0
+)
+
+// Supported returns a set of all the supported hashes by
+// HashStream and MultiHasher.
+var Supported = NewHashSet(MD5, SHA1, Dropbox, QuickXorHash)
+
+// Width returns the width in characters for any HashType
+var Width = map[Type]int{
+	MD5:          32,
+	SHA1:         40,
+	Dropbox:      64,
+	QuickXorHash: 40,
+}
+
+// Stream will calculate hashes of all supported hash types.
+func Stream(r io.Reader) (map[Type]string, error) {
+	return StreamTypes(r, Supported)
+}
+
+// StreamTypes will calculate hashes of the requested hash types.
+func StreamTypes(r io.Reader, set Set) (map[Type]string, error) {
+	hashers, err := fromTypes(set)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = io.Copy(toMultiWriter(hashers), r)
+	if err != nil {
+		return nil, err
+	}
+	var ret = make(map[Type]string)
+	for k, v := range hashers {
+		ret[k] = hex.EncodeToString(v.Sum(nil))
+	}
+	return ret, nil
+}
+
+// String returns a string representation of the hash type.
+// The function will panic if the hash type is unknown.
+func (h Type) String() string {
+	switch h {
+	case None:
+		return "None"
+	case MD5:
+		return "MD5"
+	case SHA1:
+		return "SHA-1"
+	case Dropbox:
+		return "DropboxHash"
+	case QuickXorHash:
+		return "QuickXorHash"
+	default:
+		err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h))
+		panic(err)
+	}
+}
+
+// Set a Type from a flag
+func (h *Type) Set(s string) error {
+	switch s {
+	case "None":
+		*h = None
+	case "MD5":
+		*h = MD5
+	case "SHA-1":
+		*h = SHA1
+	case "DropboxHash":
+		*h = Dropbox
+	case "QuickXorHash":
+		*h = QuickXorHash
+	default:
+		return errors.Errorf("Unknown hash type %q", s)
+	}
+	return nil
+}
+
+// Type of the value
+func (h Type) Type() string {
+	return "string"
+}
+
+// fromTypes will return hashers for all the requested types.
+// The types must be a subset of SupportedHashes,
+// and this function must support all types.
+func fromTypes(set Set) (map[Type]hash.Hash, error) {
+	if !set.SubsetOf(Supported) {
+		return nil, errors.Errorf("requested set %08x contains unknown hash types", int(set))
+	}
+	var hashers = make(map[Type]hash.Hash)
+	types := set.Array()
+	for _, t := range types {
+		switch t {
+		case MD5:
+			hashers[t] = md5.New()
+		case SHA1:
+			hashers[t] = sha1.New()
+		case Dropbox:
+			hashers[t] = dbhash.New()
+		case QuickXorHash:
+			hashers[t] = quickxorhash.New()
+		default:
+			err := fmt.Sprintf("internal error: Unsupported hash type %v", t)
+			panic(err)
+		}
+	}
+	return hashers, nil
+}
+
+// toMultiWriter will return a set of hashers into a
+// single multiwriter, where one write will update all
+// the hashers.
+func toMultiWriter(h map[Type]hash.Hash) io.Writer {
+	// Convert to to slice
+	var w = make([]io.Writer, 0, len(h))
+	for _, v := range h {
+		w = append(w, v)
+	}
+	return io.MultiWriter(w...)
+}
+
+// A MultiHasher will construct various hashes on
+// all incoming writes.
+type MultiHasher struct {
+	w    io.Writer
+	size int64
+	h    map[Type]hash.Hash // Hashes
+}
+
+// NewMultiHasher will return a hash writer that will write all
+// supported hash types.
+func NewMultiHasher() *MultiHasher {
+	h, err := NewMultiHasherTypes(Supported)
+	if err != nil {
+		panic("internal error: could not create multihasher")
+	}
+	return h
+}
+
+// NewMultiHasherTypes will return a hash writer that will write
+// the requested hash types.
+func NewMultiHasherTypes(set Set) (*MultiHasher, error) {
+	hashers, err := fromTypes(set)
+	if err != nil {
+		return nil, err
+	}
+	m := MultiHasher{h: hashers, w: toMultiWriter(hashers)}
+	return &m, nil
+}
+
+func (m *MultiHasher) Write(p []byte) (n int, err error) {
+	n, err = m.w.Write(p)
+	m.size += int64(n)
+	return n, err
+}
+
+// Sums returns the sums of all accumulated hashes as hex encoded
+// strings.
+func (m *MultiHasher) Sums() map[Type]string {
+	dst := make(map[Type]string)
+	for k, v := range m.h {
+		dst[k] = hex.EncodeToString(v.Sum(nil))
+	}
+	return dst
+}
+
+// Size returns the number of bytes written
+func (m *MultiHasher) Size() int64 {
+	return m.size
+}
+
+// A Set Indicates one or more hash types.
+type Set int
+
+// NewHashSet will create a new hash set with the hash types supplied
+func NewHashSet(t ...Type) Set {
+	h := Set(None)
+	return h.Add(t...)
+}
+
+// Add one or more hash types to the set.
+// Returns the modified hash set.
+func (h *Set) Add(t ...Type) Set {
+	for _, v := range t {
+		*h |= Set(v)
+	}
+	return *h
+}
+
+// Contains returns true if the
+func (h Set) Contains(t Type) bool {
+	return int(h)&int(t) != 0
+}
+
+// Overlap returns the overlapping hash types
+func (h Set) Overlap(t Set) Set {
+	return Set(int(h) & int(t))
+}
+
+// SubsetOf will return true if all types of h
+// is present in the set c
+func (h Set) SubsetOf(c Set) bool {
+	return int(h)|int(c) == int(c)
+}
+
+// GetOne will return a hash type.
+// Currently the first is returned, but it could be
+// improved to return the strongest.
+func (h Set) GetOne() Type {
+	v := int(h)
+	i := uint(0)
+	for v != 0 {
+		if v&1 != 0 {
+			return Type(1 << i)
+		}
+		i++
+		v >>= 1
+	}
+	return Type(None)
+}
+
+// Array returns an array of all hash types in the set
+func (h Set) Array() (ht []Type) {
+	v := int(h)
+	i := uint(0)
+	for v != 0 {
+		if v&1 != 0 {
+			ht = append(ht, Type(1<<i))
+		}
+		i++
+		v >>= 1
+	}
+	return ht
+}
+
+// Count returns the number of hash types in the set
+func (h Set) Count() int {
+	if int(h) == 0 {
+		return 0
+	}
+	// credit: https://code.google.com/u/arnehormann/
+	x := uint64(h)
+	x -= (x >> 1) & 0x5555555555555555
+	x = (x>>2)&0x3333333333333333 + x&0x3333333333333333
+	x += x >> 4
+	x &= 0x0f0f0f0f0f0f0f0f
+	x *= 0x0101010101010101
+	return int(x >> 56)
+}
+
+// String returns a string representation of the hash set.
+// The function will panic if it contains an unknown type.
+func (h Set) String() string {
+	a := h.Array()
+	var r []string
+	for _, v := range a {
+		r = append(r, v.String())
+	}
+	return "[" + strings.Join(r, ", ") + "]"
+}
+
+// Equals checks to see if src == dst, but ignores empty strings
+// and returns true if either is empty.
+func Equals(src, dst string) bool {
+	if src == "" || dst == "" {
+		return true
+	}
+	return src == dst
+}

vendor/github.com/ncw/rclone/fs/list/list.go

@@ -0,0 +1,102 @@
+// Package list contains list functions
+package list
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/filter"
+	"github.com/pkg/errors"
+)
+
+// DirSorted reads Object and *Dir into entries for the given Fs.
+//
+// dir is the start directory, "" for root
+//
+// If includeAll is specified all files will be added, otherwise only
+// files and directories passing the filter will be added.
+//
+// Files will be returned in sorted order
+func DirSorted(f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
+	// Get unfiltered entries from the fs
+	entries, err = f.List(dir)
+	if err != nil {
+		return nil, err
+	}
+	// This should happen only if exclude files lives in the
+	// starting directory, otherwise ListDirSorted should not be
+	// called.
+	if !includeAll && filter.Active.ListContainsExcludeFile(entries) {
+		fs.Debugf(dir, "Excluded from sync (and deletion)")
+		return nil, nil
+	}
+	return filterAndSortDir(entries, includeAll, dir, filter.Active.IncludeObject, filter.Active.IncludeDirectory(f))
+}
+
+// filter (if required) and check the entries, then sort them
+func filterAndSortDir(entries fs.DirEntries, includeAll bool, dir string,
+	IncludeObject func(o fs.Object) bool,
+	IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) {
+	newEntries = entries[:0] // in place filter
+	prefix := ""
+	if dir != "" {
+		prefix = dir + "/"
+	}
+	for _, entry := range entries {
+		ok := true
+		// check includes and types
+		switch x := entry.(type) {
+		case fs.Object:
+			// Make sure we don't delete excluded files if not required
+			if !includeAll && !IncludeObject(x) {
+				ok = false
+				fs.Debugf(x, "Excluded from sync (and deletion)")
+			}
+		case fs.Directory:
+			if !includeAll {
+				include, err := IncludeDirectory(x.Remote())
+				if err != nil {
+					return nil, err
+				}
+				if !include {
+					ok = false
+					fs.Debugf(x, "Excluded from sync (and deletion)")
+				}
+			}
+		default:
+			return nil, errors.Errorf("unknown object type %T", entry)
+		}
+		// check remote name belongs in this directry
+		remote := entry.Remote()
+		switch {
+		case !ok:
+			// ignore
+		case !strings.HasPrefix(remote, prefix):
+			ok = false
+			fs.Errorf(entry, "Entry doesn't belong in directory %q (too short) - ignoring", dir)
+		case remote == prefix:
+			ok = false
+			fs.Errorf(entry, "Entry doesn't belong in directory %q (same as directory) - ignoring", dir)
+		case strings.ContainsRune(remote[len(prefix):], '/'):
+			ok = false
+			fs.Errorf(entry, "Entry doesn't belong in directory %q (contains subdir) - ignoring", dir)
+		default:
+			// ok
+		}
+		if ok {
+			newEntries = append(newEntries, entry)
+		}
+	}
+	entries = newEntries
+
+	// Sort the directory entries by Remote
+	//
+	// We use a stable sort here just in case there are
+	// duplicates. Assuming the remote delivers the entries in a
+	// consistent order, this will give the best user experience
+	// in syncing as it will use the first entry for the sync
+	// comparison.
+	sort.Stable(entries)
+	return entries, nil
+}

vendor/github.com/ncw/rclone/fs/log.go

@@ -0,0 +1,135 @@
+package fs
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/pkg/errors"
+)
+
+// LogLevel describes rclone's logs.  These are a subset of the syslog log levels.
+type LogLevel byte
+
+// Log levels.  These are the syslog levels of which we only use a
+// subset.
+//
+//    LOG_EMERG      system is unusable
+//    LOG_ALERT      action must be taken immediately
+//    LOG_CRIT       critical conditions
+//    LOG_ERR        error conditions
+//    LOG_WARNING    warning conditions
+//    LOG_NOTICE     normal, but significant, condition
+//    LOG_INFO       informational message
+//    LOG_DEBUG      debug-level message
+const (
+	LogLevelEmergency LogLevel = iota
+	LogLevelAlert
+	LogLevelCritical
+	LogLevelError // Error - can't be suppressed
+	LogLevelWarning
+	LogLevelNotice // Normal logging, -q suppresses
+	LogLevelInfo   // Transfers, needs -v
+	LogLevelDebug  // Debug level, needs -vv
+)
+
+var logLevelToString = []string{
+	LogLevelEmergency: "EMERGENCY",
+	LogLevelAlert:     "ALERT",
+	LogLevelCritical:  "CRITICAL",
+	LogLevelError:     "ERROR",
+	LogLevelWarning:   "WARNING",
+	LogLevelNotice:    "NOTICE",
+	LogLevelInfo:      "INFO",
+	LogLevelDebug:     "DEBUG",
+}
+
+// String turns a LogLevel into a string
+func (l LogLevel) String() string {
+	if l >= LogLevel(len(logLevelToString)) {
+		return fmt.Sprintf("LogLevel(%d)", l)
+	}
+	return logLevelToString[l]
+}
+
+// Set a LogLevel
+func (l *LogLevel) Set(s string) error {
+	for n, name := range logLevelToString {
+		if s != "" && name == s {
+			*l = LogLevel(n)
+			return nil
+		}
+	}
+	return errors.Errorf("Unknown log level %q", s)
+}
+
+// Type of the value
+func (l *LogLevel) Type() string {
+	return "string"
+}
+
+// LogPrint sends the text to the logger of level
+var LogPrint = func(level LogLevel, text string) {
+	text = fmt.Sprintf("%-6s: %s", level, text)
+	log.Print(text)
+}
+
+// LogPrintf produces a log string from the arguments passed in
+func LogPrintf(level LogLevel, o interface{}, text string, args ...interface{}) {
+	out := fmt.Sprintf(text, args...)
+	if o != nil {
+		out = fmt.Sprintf("%v: %s", o, out)
+	}
+	LogPrint(level, out)
+}
+
+// LogLevelPrintf writes logs at the given level
+func LogLevelPrintf(level LogLevel, o interface{}, text string, args ...interface{}) {
+	if Config.LogLevel >= level {
+		LogPrintf(level, o, text, args...)
+	}
+}
+
+// Errorf writes error log output for this Object or Fs.  It
+// should always be seen by the user.
+func Errorf(o interface{}, text string, args ...interface{}) {
+	if Config.LogLevel >= LogLevelError {
+		LogPrintf(LogLevelError, o, text, args...)
+	}
+}
+
+// Logf writes log output for this Object or Fs.  This should be
+// considered to be Info level logging.  It is the default level.  By
+// default rclone should not log very much so only use this for
+// important things the user should see.  The user can filter these
+// out with the -q flag.
+func Logf(o interface{}, text string, args ...interface{}) {
+	if Config.LogLevel >= LogLevelNotice {
+		LogPrintf(LogLevelNotice, o, text, args...)
+	}
+}
+
+// Infof writes info on transfers for this Object or Fs.  Use this
+// level for logging transfers, deletions and things which should
+// appear with the -v flag.
+func Infof(o interface{}, text string, args ...interface{}) {
+	if Config.LogLevel >= LogLevelInfo {
+		LogPrintf(LogLevelInfo, o, text, args...)
+	}
+}
+
+// Debugf writes debugging output for this Object or Fs.  Use this for
+// debug only.  The user must have to specify -vv to see this.
+func Debugf(o interface{}, text string, args ...interface{}) {
+	if Config.LogLevel >= LogLevelDebug {
+		LogPrintf(LogLevelDebug, o, text, args...)
+	}
+}
+
+// LogDirName returns an object for the logger, logging a root
+// directory which would normally be "" as the Fs
+func LogDirName(f Fs, dir string) interface{} {
+	if dir != "" {
+		return dir
+	}
+	return f
+}

vendor/github.com/ncw/rclone/fs/mimetype.go

@@ -0,0 +1,44 @@
+package fs
+
+import (
+	"mime"
+	"path"
+	"strings"
+)
+
+// MimeTypeFromName returns a guess at the mime type from the name
+func MimeTypeFromName(remote string) (mimeType string) {
+	mimeType = mime.TypeByExtension(path.Ext(remote))
+	if !strings.ContainsRune(mimeType, '/') {
+		mimeType = "application/octet-stream"
+	}
+	return mimeType
+}
+
+// MimeType returns the MimeType from the object, either by calling
+// the MimeTyper interface or using MimeTypeFromName
+func MimeType(o ObjectInfo) (mimeType string) {
+	// Read the MimeType from the optional interface if available
+	if do, ok := o.(MimeTyper); ok {
+		mimeType = do.MimeType()
+		// Debugf(o, "Read MimeType as %q", mimeType)
+		if mimeType != "" {
+			return mimeType
+		}
+	}
+	return MimeTypeFromName(o.Remote())
+}
+
+// MimeTypeDirEntry returns the MimeType of a DirEntry
+//
+// It returns "inode/directory" for directories, or uses
+// MimeType(Object)
+func MimeTypeDirEntry(item DirEntry) string {
+	switch x := item.(type) {
+	case Object:
+		return MimeType(x)
+	case Directory:
+		return "inode/directory"
+	}
+	return ""
+}

vendor/github.com/ncw/rclone/fs/object/object.go

@@ -0,0 +1,239 @@
+// Package object defines some useful Objects
+package object
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"io/ioutil"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/hash"
+)
+
+// NewStaticObjectInfo returns a static ObjectInfo
+// If hashes is nil and fs is not nil, the hash map will be replaced with
+// empty hashes of the types supported by the fs.
+func NewStaticObjectInfo(remote string, modTime time.Time, size int64, storable bool, hashes map[hash.Type]string, fs fs.Info) fs.ObjectInfo {
+	info := &staticObjectInfo{
+		remote:   remote,
+		modTime:  modTime,
+		size:     size,
+		storable: storable,
+		hashes:   hashes,
+		fs:       fs,
+	}
+	if fs != nil && hashes == nil {
+		set := fs.Hashes().Array()
+		info.hashes = make(map[hash.Type]string)
+		for _, ht := range set {
+			info.hashes[ht] = ""
+		}
+	}
+	return info
+}
+
+type staticObjectInfo struct {
+	remote   string
+	modTime  time.Time
+	size     int64
+	storable bool
+	hashes   map[hash.Type]string
+	fs       fs.Info
+}
+
+func (i *staticObjectInfo) Fs() fs.Info        { return i.fs }
+func (i *staticObjectInfo) Remote() string     { return i.remote }
+func (i *staticObjectInfo) String() string     { return i.remote }
+func (i *staticObjectInfo) ModTime() time.Time { return i.modTime }
+func (i *staticObjectInfo) Size() int64        { return i.size }
+func (i *staticObjectInfo) Storable() bool     { return i.storable }
+func (i *staticObjectInfo) Hash(h hash.Type) (string, error) {
+	if len(i.hashes) == 0 {
+		return "", hash.ErrUnsupported
+	}
+	if hash, ok := i.hashes[h]; ok {
+		return hash, nil
+	}
+	return "", hash.ErrUnsupported
+}
+
+// MemoryFs is an in memory Fs, it only supports FsInfo and Put
+var MemoryFs memoryFs
+
+// memoryFs is an in memory fs
+type memoryFs struct{}
+
+// Name of the remote (as passed into NewFs)
+func (memoryFs) Name() string { return "memory" }
+
+// Root of the remote (as passed into NewFs)
+func (memoryFs) Root() string { return "" }
+
+// String returns a description of the FS
+func (memoryFs) String() string { return "memory" }
+
+// Precision of the ModTimes in this Fs
+func (memoryFs) Precision() time.Duration { return time.Nanosecond }
+
+// Returns the supported hash types of the filesystem
+func (memoryFs) Hashes() hash.Set { return hash.Supported }
+
+// Features returns the optional features of this Fs
+func (memoryFs) Features() *fs.Features { return &fs.Features{} }
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (memoryFs) List(dir string) (entries fs.DirEntries, err error) {
+	return nil, nil
+}
+
+// NewObject finds the Object at remote.  If it can't be found
+// it returns the error ErrorObjectNotFound.
+func (memoryFs) NewObject(remote string) (fs.Object, error) {
+	return nil, fs.ErrorObjectNotFound
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (memoryFs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	o := NewMemoryObject(src.Remote(), src.ModTime(), nil)
+	return o, o.Update(in, src, options...)
+}
+
+// Mkdir makes the directory (container, bucket)
+//
+// Shouldn't return an error if it already exists
+func (memoryFs) Mkdir(dir string) error {
+	return errors.New("memoryFs: can't make directory")
+}
+
+// Rmdir removes the directory (container, bucket) if empty
+//
+// Return an error if it doesn't exist or isn't empty
+func (memoryFs) Rmdir(dir string) error {
+	return fs.ErrorDirNotFound
+}
+
+var _ fs.Fs = MemoryFs
+
+// MemoryObject is an in memory object
+type MemoryObject struct {
+	remote  string
+	modTime time.Time
+	content []byte
+}
+
+// NewMemoryObject returns an in memory Object with the modTime and content passed in
+func NewMemoryObject(remote string, modTime time.Time, content []byte) *MemoryObject {
+	return &MemoryObject{
+		remote:  remote,
+		modTime: modTime,
+		content: content,
+	}
+}
+
+// Content returns the underlying buffer
+func (o *MemoryObject) Content() []byte {
+	return o.content
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (o *MemoryObject) Fs() fs.Info {
+	return MemoryFs
+}
+
+// Remote returns the remote path
+func (o *MemoryObject) Remote() string {
+	return o.remote
+}
+
+// String returns a description of the Object
+func (o *MemoryObject) String() string {
+	return o.remote
+}
+
+// ModTime returns the modification date of the file
+func (o *MemoryObject) ModTime() time.Time {
+	return o.modTime
+}
+
+// Size returns the size of the file
+func (o *MemoryObject) Size() int64 {
+	return int64(len(o.content))
+}
+
+// Storable says whether this object can be stored
+func (o *MemoryObject) Storable() bool {
+	return true
+}
+
+// Hash returns the requested hash of the contents
+func (o *MemoryObject) Hash(h hash.Type) (string, error) {
+	hash, err := hash.NewMultiHasherTypes(hash.Set(h))
+	if err != nil {
+		return "", err
+	}
+	_, err = hash.Write(o.content)
+	if err != nil {
+		return "", err
+	}
+	return hash.Sums()[h], nil
+}
+
+// SetModTime sets the metadata on the object to set the modification date
+func (o *MemoryObject) SetModTime(modTime time.Time) error {
+	o.modTime = modTime
+	return nil
+}
+
+// Open opens the file for read.  Call Close() on the returned io.ReadCloser
+func (o *MemoryObject) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
+	content := o.content
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.RangeOption:
+			content = o.content[x.Start:x.End]
+		case *fs.SeekOption:
+			content = o.content[x.Offset:]
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+	return ioutil.NopCloser(bytes.NewBuffer(content)), nil
+}
+
+// Update in to the object with the modTime given of the given size
+//
+// This re-uses the internal buffer if at all possible.
+func (o *MemoryObject) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+	size := src.Size()
+	if size == 0 {
+		o.content = nil
+	} else if size < 0 || int64(cap(o.content)) < size {
+		o.content, err = ioutil.ReadAll(in)
+	} else {
+		o.content = o.content[:size]
+		_, err = io.ReadFull(in, o.content)
+	}
+	o.modTime = src.ModTime()
+	return err
+}
+
+// Remove this object
+func (o *MemoryObject) Remove() error {
+	return errors.New("memoryObject.Remove not supported")
+}

vendor/github.com/ncw/rclone/fs/options.go

@@ -0,0 +1,262 @@
+// Define the options for Open
+
+package fs
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/pkg/errors"
+)
+
+// OpenOption is an interface describing options for Open
+type OpenOption interface {
+	fmt.Stringer
+
+	// Header returns the option as an HTTP header
+	Header() (key string, value string)
+
+	// Mandatory returns whether this option can be ignored or not
+	Mandatory() bool
+}
+
+// RangeOption defines an HTTP Range option with start and end.  If
+// either start or end are < 0 then they will be omitted.
+//
+// End may be bigger than the Size of the object in which case it will
+// be capped to the size of the object.
+//
+// Note that the End is inclusive, so to fetch 100 bytes you would use
+// RangeOption{Start: 0, End: 99}
+//
+// If Start is specified but End is not then it will fetch from Start
+// to the end of the file.
+//
+// If End is specified, but Start is not then it will fetch the last
+// End bytes.
+//
+// Examples:
+//
+//     RangeOption{Start: 0, End: 99} - fetch the first 100 bytes
+//     RangeOption{Start: 100, End: 199} - fetch the second 100 bytes
+//     RangeOption{Start: 100} - fetch bytes from offset 100 to the end
+//     RangeOption{End: 100} - fetch the last 100 bytes
+//
+// A RangeOption implements a single byte-range-spec from
+// https://tools.ietf.org/html/rfc7233#section-2.1
+type RangeOption struct {
+	Start int64
+	End   int64
+}
+
+// Header formats the option as an http header
+func (o *RangeOption) Header() (key string, value string) {
+	key = "Range"
+	value = "bytes="
+	if o.Start >= 0 {
+		value += strconv.FormatInt(o.Start, 10)
+
+	}
+	value += "-"
+	if o.End >= 0 {
+		value += strconv.FormatInt(o.End, 10)
+	}
+	return key, value
+}
+
+// ParseRangeOption parses a RangeOption from a Range: header.
+// It only appects single ranges.
+func ParseRangeOption(s string) (po *RangeOption, err error) {
+	const preamble = "bytes="
+	if !strings.HasPrefix(s, preamble) {
+		return nil, errors.New("Range: header invalid: doesn't start with " + preamble)
+	}
+	s = s[len(preamble):]
+	if strings.IndexRune(s, ',') >= 0 {
+		return nil, errors.New("Range: header invalid: contains multiple ranges which isn't supported")
+	}
+	dash := strings.IndexRune(s, '-')
+	if dash < 0 {
+		return nil, errors.New("Range: header invalid: contains no '-'")
+	}
+	start, end := strings.TrimSpace(s[:dash]), strings.TrimSpace(s[dash+1:])
+	o := RangeOption{Start: -1, End: -1}
+	if start != "" {
+		o.Start, err = strconv.ParseInt(start, 10, 64)
+		if err != nil || o.Start < 0 {
+			return nil, errors.New("Range: header invalid: bad start")
+		}
+	}
+	if end != "" {
+		o.End, err = strconv.ParseInt(end, 10, 64)
+		if err != nil || o.End < 0 {
+			return nil, errors.New("Range: header invalid: bad end")
+		}
+	}
+	return &o, nil
+}
+
+// String formats the option into human readable form
+func (o *RangeOption) String() string {
+	return fmt.Sprintf("RangeOption(%d,%d)", o.Start, o.End)
+}
+
+// Mandatory returns whether the option must be parsed or can be ignored
+func (o *RangeOption) Mandatory() bool {
+	return true
+}
+
+// Decode interprets the RangeOption into an offset and a limit
+//
+// The offset is the start of the stream and the limit is how many
+// bytes should be read from it.  If the limit is -1 then the stream
+// should be read to the end.
+func (o *RangeOption) Decode(size int64) (offset, limit int64) {
+	if o.Start >= 0 {
+		offset = o.Start
+		if o.End >= 0 {
+			limit = o.End - o.Start + 1
+		} else {
+			limit = -1
+		}
+	} else {
+		if o.End >= 0 {
+			offset = size - o.End
+		} else {
+			offset = 0
+		}
+		limit = -1
+	}
+	return offset, limit
+}
+
+// FixRangeOption looks through the slice of options and adjusts any
+// RangeOption~s found that request a fetch from the end into an
+// absolute fetch using the size passed in and makes sure the range does
+// not exceed filesize. Some remotes (eg Onedrive, Box) don't support
+// range requests which index from the end.
+func FixRangeOption(options []OpenOption, size int64) {
+	for i := range options {
+		option := options[i]
+		if x, ok := option.(*RangeOption); ok {
+			// If start is < 0 then fetch from the end
+			if x.Start < 0 {
+				x = &RangeOption{Start: size - x.End, End: -1}
+				options[i] = x
+			}
+			if x.End > size {
+				x = &RangeOption{Start: x.Start, End: size}
+				options[i] = x
+			}
+		}
+	}
+}
+
+// SeekOption defines an HTTP Range option with start only.
+type SeekOption struct {
+	Offset int64
+}
+
+// Header formats the option as an http header
+func (o *SeekOption) Header() (key string, value string) {
+	key = "Range"
+	value = fmt.Sprintf("bytes=%d-", o.Offset)
+	return key, value
+}
+
+// String formats the option into human readable form
+func (o *SeekOption) String() string {
+	return fmt.Sprintf("SeekOption(%d)", o.Offset)
+}
+
+// Mandatory returns whether the option must be parsed or can be ignored
+func (o *SeekOption) Mandatory() bool {
+	return true
+}
+
+// HTTPOption defines a general purpose HTTP option
+type HTTPOption struct {
+	Key   string
+	Value string
+}
+
+// Header formats the option as an http header
+func (o *HTTPOption) Header() (key string, value string) {
+	return o.Key, o.Value
+}
+
+// String formats the option into human readable form
+func (o *HTTPOption) String() string {
+	return fmt.Sprintf("HTTPOption(%q,%q)", o.Key, o.Value)
+}
+
+// Mandatory returns whether the option must be parsed or can be ignored
+func (o *HTTPOption) Mandatory() bool {
+	return false
+}
+
+// HashesOption defines an option used to tell the local fs to limit
+// the number of hashes it calculates.
+type HashesOption struct {
+	Hashes hash.Set
+}
+
+// Header formats the option as an http header
+func (o *HashesOption) Header() (key string, value string) {
+	return "", ""
+}
+
+// String formats the option into human readable form
+func (o *HashesOption) String() string {
+	return fmt.Sprintf("HashesOption(%v)", o.Hashes)
+}
+
+// Mandatory returns whether the option must be parsed or can be ignored
+func (o *HashesOption) Mandatory() bool {
+	return false
+}
+
+// OpenOptionAddHeaders adds each header found in options to the
+// headers map provided the key was non empty.
+func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {
+	for _, option := range options {
+		key, value := option.Header()
+		if key != "" && value != "" {
+			headers[key] = value
+		}
+	}
+}
+
+// OpenOptionHeaders adds each header found in options to the
+// headers map provided the key was non empty.
+//
+// It returns a nil map if options was empty
+func OpenOptionHeaders(options []OpenOption) (headers map[string]string) {
+	if len(options) == 0 {
+		return nil
+	}
+	headers = make(map[string]string, len(options))
+	OpenOptionAddHeaders(options, headers)
+	return headers
+}
+
+// OpenOptionAddHTTPHeaders Sets each header found in options to the
+// http.Header map provided the key was non empty.
+func OpenOptionAddHTTPHeaders(headers http.Header, options []OpenOption) {
+	for _, option := range options {
+		key, value := option.Header()
+		if key != "" && value != "" {
+			headers.Set(key, value)
+		}
+	}
+}
+
+// check interface
+var (
+	_ OpenOption = (*RangeOption)(nil)
+	_ OpenOption = (*SeekOption)(nil)
+	_ OpenOption = (*HTTPOption)(nil)
+)

vendor/github.com/ncw/rclone/fs/parseduration.go

@@ -0,0 +1,103 @@
+package fs
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Duration is a time.Duration with some more parsing options
+type Duration time.Duration
+
+// DurationOff is the default value for flags which can be turned off
+const DurationOff = Duration((1 << 63) - 1)
+
+// Turn Duration into a string
+func (d Duration) String() string {
+	if d == DurationOff {
+		return "off"
+	}
+	for i := len(ageSuffixes) - 2; i >= 0; i-- {
+		ageSuffix := &ageSuffixes[i]
+		if math.Abs(float64(d)) >= float64(ageSuffix.Multiplier) {
+			timeUnits := float64(d) / float64(ageSuffix.Multiplier)
+			return strconv.FormatFloat(timeUnits, 'f', -1, 64) + ageSuffix.Suffix
+		}
+	}
+	return time.Duration(d).String()
+}
+
+// IsSet returns if the duration is != DurationOff
+func (d Duration) IsSet() bool {
+	return d != DurationOff
+}
+
+// We use time conventions
+var ageSuffixes = []struct {
+	Suffix     string
+	Multiplier time.Duration
+}{
+	{Suffix: "d", Multiplier: time.Hour * 24},
+	{Suffix: "w", Multiplier: time.Hour * 24 * 7},
+	{Suffix: "M", Multiplier: time.Hour * 24 * 30},
+	{Suffix: "y", Multiplier: time.Hour * 24 * 365},
+
+	// Default to second
+	{Suffix: "", Multiplier: time.Second},
+}
+
+// ParseDuration parses a duration string. Accept ms|s|m|h|d|w|M|y suffixes. Defaults to second if not provided
+func ParseDuration(age string) (time.Duration, error) {
+	var period float64
+
+	if age == "off" {
+		return time.Duration(DurationOff), nil
+	}
+
+	// Attempt to parse as a time.Duration first
+	d, err := time.ParseDuration(age)
+	if err == nil {
+		return d, nil
+	}
+
+	for _, ageSuffix := range ageSuffixes {
+		if strings.HasSuffix(age, ageSuffix.Suffix) {
+			numberString := age[:len(age)-len(ageSuffix.Suffix)]
+			var err error
+			period, err = strconv.ParseFloat(numberString, 64)
+			if err != nil {
+				return time.Duration(0), err
+			}
+			period *= float64(ageSuffix.Multiplier)
+			break
+		}
+	}
+
+	return time.Duration(period), nil
+}
+
+// Set a Duration
+func (d *Duration) Set(s string) error {
+	duration, err := ParseDuration(s)
+	if err != nil {
+		return err
+	}
+	*d = Duration(duration)
+	return nil
+}
+
+// Type of the value
+func (d Duration) Type() string {
+	return "duration"
+}
+
+// Scan implements the fmt.Scanner interface
+func (d *Duration) Scan(s fmt.ScanState, ch rune) error {
+	token, err := s.Token(true, nil)
+	if err != nil {
+		return err
+	}
+	return d.Set(string(token))
+}

vendor/github.com/ncw/rclone/fs/rc/internal.go

@@ -0,0 +1,131 @@
+// Define the internal rc functions
+
+package rc
+
+import (
+	"os"
+	"runtime"
+
+	"github.com/pkg/errors"
+)
+
+func init() {
+	Add(Call{
+		Path:  "rc/noop",
+		Fn:    rcNoop,
+		Title: "Echo the input to the output parameters",
+		Help: `
+This echoes the input parameters to the output parameters for testing
+purposes.  It can be used to check that rclone is still alive and to
+check that parameter passing is working properly.`,
+	})
+	Add(Call{
+		Path:  "rc/error",
+		Fn:    rcError,
+		Title: "This returns an error",
+		Help: `
+This returns an error with the input as part of its error string.
+Useful for testing error handling.`,
+	})
+	Add(Call{
+		Path:  "rc/list",
+		Fn:    rcList,
+		Title: "List all the registered remote control commands",
+		Help: `
+This lists all the registered remote control commands as a JSON map in
+the commands response.`,
+	})
+	Add(Call{
+		Path:  "core/pid",
+		Fn:    rcPid,
+		Title: "Return PID of current process",
+		Help: `
+This returns PID of current process.
+Useful for stopping rclone process.`,
+	})
+	Add(Call{
+		Path:  "core/memstats",
+		Fn:    rcMemStats,
+		Title: "Returns the memory statistics",
+		Help: `
+This returns the memory statistics of the running program.  What the values mean
+are explained in the go docs: https://golang.org/pkg/runtime/#MemStats
+
+The most interesting values for most people are:
+
+* HeapAlloc: This is the amount of memory rclone is actually using
+* HeapSys: This is the amount of memory rclone has obtained from the OS
+* Sys: this is the total amount of memory requested from the OS
+  * It is virtual memory so may include unused memory
+`,
+	})
+	Add(Call{
+		Path:  "core/gc",
+		Fn:    rcGc,
+		Title: "Runs a garbage collection.",
+		Help: `
+This tells the go runtime to do a garbage collection run.  It isn't
+necessary to call this normally, but it can be useful for debugging
+memory problems.
+`,
+	})
+}
+
+// Echo the input to the ouput parameters
+func rcNoop(in Params) (out Params, err error) {
+	return in, nil
+}
+
+// Return an error regardless
+func rcError(in Params) (out Params, err error) {
+	return nil, errors.Errorf("arbitrary error on input %+v", in)
+}
+
+// List the registered commands
+func rcList(in Params) (out Params, err error) {
+	out = make(Params)
+	out["commands"] = registry.list()
+	return out, nil
+}
+
+// Return PID of current process
+func rcPid(in Params) (out Params, err error) {
+	out = make(Params)
+	out["pid"] = os.Getpid()
+	return out, nil
+}
+
+// Return the memory statistics
+func rcMemStats(in Params) (out Params, err error) {
+	out = make(Params)
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	out["Alloc"] = m.Alloc
+	out["TotalAlloc"] = m.TotalAlloc
+	out["Sys"] = m.Sys
+	out["Mallocs"] = m.Mallocs
+	out["Frees"] = m.Frees
+	out["HeapAlloc"] = m.HeapAlloc
+	out["HeapSys"] = m.HeapSys
+	out["HeapIdle"] = m.HeapIdle
+	out["HeapInuse"] = m.HeapInuse
+	out["HeapReleased"] = m.HeapReleased
+	out["HeapObjects"] = m.HeapObjects
+	out["StackInuse"] = m.StackInuse
+	out["StackSys"] = m.StackSys
+	out["MSpanInuse"] = m.MSpanInuse
+	out["MSpanSys"] = m.MSpanSys
+	out["MCacheInuse"] = m.MCacheInuse
+	out["MCacheSys"] = m.MCacheSys
+	out["BuckHashSys"] = m.BuckHashSys
+	out["GCSys"] = m.GCSys
+	out["OtherSys"] = m.OtherSys
+	return out, nil
+}
+
+// Do a garbage collection run
+func rcGc(in Params) (out Params, err error) {
+	out = make(Params)
+	runtime.GC()
+	return out, nil
+}

vendor/github.com/ncw/rclone/fs/rc/rc.go

@@ -0,0 +1,146 @@
+// Package rc implements a remote control server and registry for rclone
+//
+// To register your internal calls, call rc.Add(path, function).  Your
+// function should take ane return a Param.  It can also return an
+// error.  Use rc.NewError to wrap an existing error along with an
+// http response type if another response other than 500 internal
+// error is required on error.
+package rc
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	_ "net/http/pprof" // install the pprof http handlers
+
+	"strings"
+
+	"github.com/ncw/rclone/cmd/serve/httplib"
+	"github.com/ncw/rclone/fs"
+	"github.com/pkg/errors"
+)
+
+// Options contains options for the remote control server
+type Options struct {
+	HTTPOptions httplib.Options
+	Enabled     bool
+}
+
+// DefaultOpt is the default values used for Options
+var DefaultOpt = Options{
+	HTTPOptions: httplib.DefaultOpt,
+	Enabled:     false,
+}
+
+func init() {
+	DefaultOpt.HTTPOptions.ListenAddr = "localhost:5572"
+}
+
+// Start the remote control server if configured
+func Start(opt *Options) {
+	if opt.Enabled {
+		s := newServer(opt)
+		go s.serve()
+	}
+}
+
+// server contains everything to run the server
+type server struct {
+	srv *httplib.Server
+}
+
+func newServer(opt *Options) *server {
+	// Serve on the DefaultServeMux so can have global registrations appear
+	mux := http.DefaultServeMux
+	s := &server{
+		srv: httplib.NewServer(mux, &opt.HTTPOptions),
+	}
+	mux.HandleFunc("/", s.handler)
+	return s
+}
+
+// serve runs the http server - doesn't return
+func (s *server) serve() {
+	err := s.srv.Serve()
+	if err != nil {
+		fs.Errorf(nil, "Opening listener: %v", err)
+	}
+	fs.Logf(nil, "Serving remote control on %s", s.srv.URL())
+	s.srv.Wait()
+}
+
+// WriteJSON writes JSON in out to w
+func WriteJSON(w io.Writer, out Params) error {
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "\t")
+	return enc.Encode(out)
+}
+
+// handler reads incoming requests and dispatches them
+func (s *server) handler(w http.ResponseWriter, r *http.Request) {
+	path := strings.Trim(r.URL.Path, "/")
+	in := make(Params)
+
+	writeError := func(err error, status int) {
+		fs.Errorf(nil, "rc: %q: error: %v", path, err)
+		w.WriteHeader(status)
+		err = WriteJSON(w, Params{
+			"error": err.Error(),
+			"input": in,
+		})
+		if err != nil {
+			// can't return the error at this point
+			fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
+		}
+	}
+
+	if r.Method != "POST" {
+		writeError(errors.Errorf("method %q not allowed - POST required", r.Method), http.StatusMethodNotAllowed)
+		return
+	}
+
+	// Find the call
+	call := registry.get(path)
+	if call == nil {
+		writeError(errors.Errorf("couldn't find method %q", path), http.StatusMethodNotAllowed)
+		return
+	}
+
+	// Parse the POST and URL parameters into r.Form
+	err := r.ParseForm()
+	if err != nil {
+		writeError(errors.Wrap(err, "failed to parse form/URL parameters"), http.StatusBadRequest)
+		return
+	}
+
+	// Read the POST and URL parameters into in
+	for k, vs := range r.Form {
+		if len(vs) > 0 {
+			in[k] = vs[len(vs)-1]
+		}
+	}
+	fs.Debugf(nil, "form = %+v", r.Form)
+
+	// Parse a JSON blob from the input
+	if r.Header.Get("Content-Type") == "application/json" {
+		err := json.NewDecoder(r.Body).Decode(&in)
+		if err != nil {
+			writeError(errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest)
+			return
+		}
+	}
+
+	fs.Debugf(nil, "rc: %q: with parameters %+v", path, in)
+	out, err := call.Fn(in)
+	if err != nil {
+		writeError(errors.Wrap(err, "remote control command failed"), http.StatusInternalServerError)
+		return
+	}
+
+	fs.Debugf(nil, "rc: %q: reply %+v: %v", path, out, err)
+	err = WriteJSON(w, out)
+	if err != nil {
+		// can't return the error at this point
+		fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
+	}
+}

vendor/github.com/ncw/rclone/fs/rc/registry.go

@@ -0,0 +1,79 @@
+// Define the registry
+
+package rc
+
+import (
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/ncw/rclone/fs"
+)
+
+// Params is the input and output type for the Func
+type Params map[string]interface{}
+
+// Func defines a type for a remote control function
+type Func func(in Params) (out Params, err error)
+
+// Call defines info about a remote control function and is used in
+// the Add function to create new entry points.
+type Call struct {
+	Path  string // path to activate this RC
+	Fn    Func   `json:"-"` // function to call
+	Title string // help for the function
+	Help  string // multi-line markdown formatted help
+}
+
+// Registry holds the list of all the registered remote control functions
+type Registry struct {
+	mu   sync.RWMutex
+	call map[string]*Call
+}
+
+// NewRegistry makes a new registry for remote control functions
+func NewRegistry() *Registry {
+	return &Registry{
+		call: make(map[string]*Call),
+	}
+}
+
+// Add a call to the registry
+func (r *Registry) add(call Call) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	call.Path = strings.Trim(call.Path, "/")
+	call.Help = strings.TrimSpace(call.Help)
+	fs.Debugf(nil, "Adding path %q to remote control registry", call.Path)
+	r.call[call.Path] = &call
+}
+
+// get a Call from a path or nil
+func (r *Registry) get(path string) *Call {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.call[path]
+}
+
+// get a list of all calls in alphabetical order
+func (r *Registry) list() (out []*Call) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	var keys []string
+	for key := range r.call {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		out = append(out, r.call[key])
+	}
+	return out
+}
+
+// The global registry
+var registry = NewRegistry()
+
+// Add a function to the global registry
+func Add(call Call) {
+	registry.add(call)
+}

vendor/github.com/ncw/rclone/fs/sizesuffix.go

@@ -0,0 +1,132 @@
+package fs
+
+// SizeSuffix is parsed by flag with k/M/G suffixes
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// SizeSuffix is an int64 with a friendly way of printing setting
+type SizeSuffix int64
+
+// Common multipliers for SizeSuffix
+const (
+	Byte SizeSuffix = 1 << (iota * 10)
+	KibiByte
+	MebiByte
+	GibiByte
+	TebiByte
+	PebiByte
+	ExbiByte
+)
+
+// Turn SizeSuffix into a string and a suffix
+func (x SizeSuffix) string() (string, string) {
+	scaled := float64(0)
+	suffix := ""
+	switch {
+	case x < 0:
+		return "off", ""
+	case x == 0:
+		return "0", ""
+	case x < 1<<10:
+		scaled = float64(x)
+		suffix = ""
+	case x < 1<<20:
+		scaled = float64(x) / (1 << 10)
+		suffix = "k"
+	case x < 1<<30:
+		scaled = float64(x) / (1 << 20)
+		suffix = "M"
+	case x < 1<<40:
+		scaled = float64(x) / (1 << 30)
+		suffix = "G"
+	case x < 1<<50:
+		scaled = float64(x) / (1 << 40)
+		suffix = "T"
+	default:
+		scaled = float64(x) / (1 << 50)
+		suffix = "P"
+	}
+	if math.Floor(scaled) == scaled {
+		return fmt.Sprintf("%.0f", scaled), suffix
+	}
+	return fmt.Sprintf("%.3f", scaled), suffix
+}
+
+// String turns SizeSuffix into a string
+func (x SizeSuffix) String() string {
+	val, suffix := x.string()
+	return val + suffix
+}
+
+// Unit turns SizeSuffix into a string with a unit
+func (x SizeSuffix) Unit(unit string) string {
+	val, suffix := x.string()
+	if val == "off" {
+		return val
+	}
+	return val + " " + suffix + unit
+}
+
+// Set a SizeSuffix
+func (x *SizeSuffix) Set(s string) error {
+	if len(s) == 0 {
+		return errors.New("empty string")
+	}
+	if strings.ToLower(s) == "off" {
+		*x = -1
+		return nil
+	}
+	suffix := s[len(s)-1]
+	suffixLen := 1
+	var multiplier float64
+	switch suffix {
+	case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.':
+		suffixLen = 0
+		multiplier = 1 << 10
+	case 'b', 'B':
+		multiplier = 1
+	case 'k', 'K':
+		multiplier = 1 << 10
+	case 'm', 'M':
+		multiplier = 1 << 20
+	case 'g', 'G':
+		multiplier = 1 << 30
+	case 't', 'T':
+		multiplier = 1 << 40
+	case 'p', 'P':
+		multiplier = 1 << 50
+	default:
+		return errors.Errorf("bad suffix %q", suffix)
+	}
+	s = s[:len(s)-suffixLen]
+	value, err := strconv.ParseFloat(s, 64)
+	if err != nil {
+		return err
+	}
+	if value < 0 {
+		return errors.Errorf("size can't be negative %q", s)
+	}
+	value *= multiplier
+	*x = SizeSuffix(value)
+	return nil
+}
+
+// Type of the value
+func (x *SizeSuffix) Type() string {
+	return "int64"
+}
+
+// Scan implements the fmt.Scanner interface
+func (x *SizeSuffix) Scan(s fmt.ScanState, ch rune) error {
+	token, err := s.Token(true, nil)
+	if err != nil {
+		return err
+	}
+	return x.Set(string(token))
+}

vendor/github.com/ncw/rclone/fs/version.go

@@ -0,0 +1,4 @@
+package fs
+
+// Version of rclone
+var Version = "v1.43.1"

vendor/github.com/ncw/rclone/fs/versioncheck.go

@@ -0,0 +1,7 @@
+//+build !go1.7
+
+package fs
+
+// Upgrade to Go version 1.7 to compile rclone - latest stable go
+// compiler recommended.
+func init() { Go_version_1_7_required_for_compilation() }

vendor/github.com/ncw/rclone/fs/walk/walk.go

@@ -0,0 +1,552 @@
+// Package walk walks directories
+package walk
+
+import (
+	"bytes"
+	"fmt"
+	"path"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/filter"
+	"github.com/ncw/rclone/fs/list"
+	"github.com/pkg/errors"
+)
+
+// ErrorSkipDir is used as a return value from Walk to indicate that the
+// directory named in the call is to be skipped. It is not returned as
+// an error by any function.
+var ErrorSkipDir = errors.New("skip this directory")
+
+// ErrorCantListR is returned by WalkR if the underlying Fs isn't
+// capable of doing a recursive listing.
+var ErrorCantListR = errors.New("recursive directory listing not available")
+
+// Func is the type of the function called for directory
+// visited by Walk. The path argument contains remote path to the directory.
+//
+// If there was a problem walking to directory named by path, the
+// incoming error will describe the problem and the function can
+// decide how to handle that error (and Walk will not descend into
+// that directory). If an error is returned, processing stops. The
+// sole exception is when the function returns the special value
+// ErrorSkipDir. If the function returns ErrorSkipDir, Walk skips the
+// directory's contents entirely.
+type Func func(path string, entries fs.DirEntries, err error) error
+
+// Walk lists the directory.
+//
+// If includeAll is not set it will use the filters defined.
+//
+// If maxLevel is < 0 then it will recurse indefinitely, else it will
+// only do maxLevel levels.
+//
+// It calls fn for each tranche of DirEntries read.
+//
+// Note that fn will not be called concurrently whereas the directory
+// listing will proceed concurrently.
+//
+// Parent directories are always listed before their children
+//
+// This is implemented by WalkR if Config.UseRecursiveListing is true
+// and f supports it and level > 1, or WalkN otherwise.
+//
+// NB (f, path) to be replaced by fs.Dir at some point
+func Walk(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
+	if (maxLevel < 0 || maxLevel > 1) && fs.Config.UseListR && f.Features().ListR != nil {
+		return walkListR(f, path, includeAll, maxLevel, fn)
+	}
+	return walkListDirSorted(f, path, includeAll, maxLevel, fn)
+}
+
+// walkListDirSorted lists the directory.
+//
+// It implements Walk using non recursive directory listing.
+func walkListDirSorted(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
+	return walk(f, path, includeAll, maxLevel, fn, list.DirSorted)
+}
+
+// walkListR lists the directory.
+//
+// It implements Walk using recursive directory listing if
+// available, or returns ErrorCantListR if not.
+func walkListR(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
+	listR := f.Features().ListR
+	if listR == nil {
+		return ErrorCantListR
+	}
+	return walkR(f, path, includeAll, maxLevel, fn, listR)
+}
+
+type listDirFunc func(fs fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error)
+
+func walk(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listDir listDirFunc) error {
+	var (
+		wg         sync.WaitGroup // sync closing of go routines
+		traversing sync.WaitGroup // running directory traversals
+		doClose    sync.Once      // close the channel once
+		mu         sync.Mutex     // stop fn being called concurrently
+	)
+	// listJob describe a directory listing that needs to be done
+	type listJob struct {
+		remote string
+		depth  int
+	}
+
+	in := make(chan listJob, fs.Config.Checkers)
+	errs := make(chan error, 1)
+	quit := make(chan struct{})
+	closeQuit := func() {
+		doClose.Do(func() {
+			close(quit)
+			go func() {
+				for range in {
+					traversing.Done()
+				}
+			}()
+		})
+	}
+	for i := 0; i < fs.Config.Checkers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case job, ok := <-in:
+					if !ok {
+						return
+					}
+					entries, err := listDir(f, includeAll, job.remote)
+					var jobs []listJob
+					if err == nil && job.depth != 0 {
+						entries.ForDir(func(dir fs.Directory) {
+							// Recurse for the directory
+							jobs = append(jobs, listJob{
+								remote: dir.Remote(),
+								depth:  job.depth - 1,
+							})
+						})
+					}
+					mu.Lock()
+					err = fn(job.remote, entries, err)
+					mu.Unlock()
+					// NB once we have passed entries to fn we mustn't touch it again
+					if err != nil && err != ErrorSkipDir {
+						traversing.Done()
+						fs.CountError(err)
+						fs.Errorf(job.remote, "error listing: %v", err)
+						closeQuit()
+						// Send error to error channel if space
+						select {
+						case errs <- err:
+						default:
+						}
+						continue
+					}
+					if err == nil && len(jobs) > 0 {
+						traversing.Add(len(jobs))
+						go func() {
+							// Now we have traversed this directory, send these
+							// jobs off for traversal in the background
+							for _, newJob := range jobs {
+								in <- newJob
+							}
+						}()
+					}
+					traversing.Done()
+				case <-quit:
+					return
+				}
+			}
+		}()
+	}
+	// Start the process
+	traversing.Add(1)
+	in <- listJob{
+		remote: path,
+		depth:  maxLevel - 1,
+	}
+	traversing.Wait()
+	close(in)
+	wg.Wait()
+	close(errs)
+	// return the first error returned or nil
+	return <-errs
+}
+
+// DirTree is a map of directories to entries
+type DirTree map[string]fs.DirEntries
+
+// parentDir finds the parent directory of path
+func parentDir(entryPath string) string {
+	dirPath := path.Dir(entryPath)
+	if dirPath == "." {
+		dirPath = ""
+	}
+	return dirPath
+}
+
+// add an entry to the tree
+func (dt DirTree) add(entry fs.DirEntry) {
+	dirPath := parentDir(entry.Remote())
+	dt[dirPath] = append(dt[dirPath], entry)
+}
+
+// add a directory entry to the tree
+func (dt DirTree) addDir(entry fs.DirEntry) {
+	dt.add(entry)
+	// create the directory itself if it doesn't exist already
+	dirPath := entry.Remote()
+	if _, ok := dt[dirPath]; !ok {
+		dt[dirPath] = nil
+	}
+}
+
+// Find returns the DirEntry for filePath or nil if not found
+func (dt DirTree) Find(filePath string) (parentPath string, entry fs.DirEntry) {
+	parentPath = parentDir(filePath)
+	for _, entry := range dt[parentPath] {
+		if entry.Remote() == filePath {
+			return parentPath, entry
+		}
+	}
+	return parentPath, nil
+}
+
+// check that dirPath has a *Dir in its parent
+func (dt DirTree) checkParent(root, dirPath string) {
+	if dirPath == root {
+		return
+	}
+	parentPath, entry := dt.Find(dirPath)
+	if entry != nil {
+		return
+	}
+	dt[parentPath] = append(dt[parentPath], fs.NewDir(dirPath, time.Now()))
+	dt.checkParent(root, parentPath)
+}
+
+// check every directory in the tree has *Dir in its parent
+func (dt DirTree) checkParents(root string) {
+	for dirPath := range dt {
+		dt.checkParent(root, dirPath)
+	}
+}
+
+// Sort sorts all the Entries
+func (dt DirTree) Sort() {
+	for _, entries := range dt {
+		sort.Stable(entries)
+	}
+}
+
+// Dirs returns the directories in sorted order
+func (dt DirTree) Dirs() (dirNames []string) {
+	for dirPath := range dt {
+		dirNames = append(dirNames, dirPath)
+	}
+	sort.Strings(dirNames)
+	return dirNames
+}
+
+// Prune remove directories from a directory tree. dirNames contains
+// all directories to remove as keys, with true as values. dirNames
+// will be modified in the function.
+func (dt DirTree) Prune(dirNames map[string]bool) error {
+	// We use map[string]bool to avoid recursion (and potential
+	// stack exhaustion).
+
+	// First we need delete directories from their parents.
+	for dName, remove := range dirNames {
+		if !remove {
+			// Currently all values should be
+			// true, therefore this should not
+			// happen. But this makes function
+			// more predictable.
+			fs.Infof(dName, "Directory in the map for prune, but the value is false")
+			continue
+		}
+		if dName == "" {
+			// if dName is root, do nothing (no parent exist)
+			continue
+		}
+		parent := parentDir(dName)
+		// It may happen that dt does not have a dName key,
+		// since directory was excluded based on a filter. In
+		// such case the loop will be skipped.
+		for i, entry := range dt[parent] {
+			switch x := entry.(type) {
+			case fs.Directory:
+				if x.Remote() == dName {
+					// the slice is not sorted yet
+					// to delete item
+					// a) replace it with the last one
+					dt[parent][i] = dt[parent][len(dt[parent])-1]
+					// b) remove last
+					dt[parent] = dt[parent][:len(dt[parent])-1]
+					// we modify a slice within a loop, but we stop
+					// iterating immediately
+					break
+				}
+			case fs.Object:
+				// do nothing
+			default:
+				return errors.Errorf("unknown object type %T", entry)
+
+			}
+		}
+	}
+
+	for len(dirNames) > 0 {
+		// According to golang specs, if new keys were added
+		// during range iteration, they may be skipped.
+		for dName, remove := range dirNames {
+			if !remove {
+				fs.Infof(dName, "Directory in the map for prune, but the value is false")
+				continue
+			}
+			// First, add all subdirectories to dirNames.
+
+			// It may happen that dt[dName] does not exist.
+			// If so, the loop will be skipped.
+			for _, entry := range dt[dName] {
+				switch x := entry.(type) {
+				case fs.Directory:
+					excludeDir := x.Remote()
+					dirNames[excludeDir] = true
+				case fs.Object:
+					// do nothing
+				default:
+					return errors.Errorf("unknown object type %T", entry)
+
+				}
+			}
+			// Then remove current directory from DirTree
+			delete(dt, dName)
+			// and from dirNames
+			delete(dirNames, dName)
+		}
+	}
+	return nil
+}
+
+// String emits a simple representation of the DirTree
+func (dt DirTree) String() string {
+	out := new(bytes.Buffer)
+	for _, dir := range dt.Dirs() {
+		_, _ = fmt.Fprintf(out, "%s/\n", dir)
+		for _, entry := range dt[dir] {
+			flag := ""
+			if _, ok := entry.(fs.Directory); ok {
+				flag = "/"
+			}
+			_, _ = fmt.Fprintf(out, "  %s%s\n", path.Base(entry.Remote()), flag)
+		}
+	}
+	return out.String()
+}
+
+func walkRDirTree(f fs.Fs, startPath string, includeAll bool, maxLevel int, listR fs.ListRFn) (DirTree, error) {
+	dirs := make(DirTree)
+	// Entries can come in arbitrary order. We use toPrune to keep
+	// all directories to exclude later.
+	toPrune := make(map[string]bool)
+	includeDirectory := filter.Active.IncludeDirectory(f)
+	var mu sync.Mutex
+	err := listR(startPath, func(entries fs.DirEntries) error {
+		mu.Lock()
+		defer mu.Unlock()
+		for _, entry := range entries {
+			slashes := strings.Count(entry.Remote(), "/")
+			switch x := entry.(type) {
+			case fs.Object:
+				// Make sure we don't delete excluded files if not required
+				if includeAll || filter.Active.IncludeObject(x) {
+					if maxLevel < 0 || slashes <= maxLevel-1 {
+						dirs.add(x)
+					} else {
+						// Make sure we include any parent directories of excluded objects
+						dirPath := x.Remote()
+						for ; slashes > maxLevel-1; slashes-- {
+							dirPath = parentDir(dirPath)
+						}
+						dirs.checkParent(startPath, dirPath)
+					}
+				} else {
+					fs.Debugf(x, "Excluded from sync (and deletion)")
+				}
+				// Check if we need to prune a directory later.
+				if !includeAll && len(filter.Active.Opt.ExcludeFile) > 0 {
+					basename := path.Base(x.Remote())
+					if basename == filter.Active.Opt.ExcludeFile {
+						excludeDir := parentDir(x.Remote())
+						toPrune[excludeDir] = true
+						fs.Debugf(basename, "Excluded from sync (and deletion) based on exclude file")
+					}
+				}
+			case fs.Directory:
+				inc, err := includeDirectory(x.Remote())
+				if err != nil {
+					return err
+				}
+				if includeAll || inc {
+					if maxLevel < 0 || slashes <= maxLevel-1 {
+						if slashes == maxLevel-1 {
+							// Just add the object if at maxLevel
+							dirs.add(x)
+						} else {
+							dirs.addDir(x)
+						}
+					}
+				} else {
+					fs.Debugf(x, "Excluded from sync (and deletion)")
+				}
+			default:
+				return errors.Errorf("unknown object type %T", entry)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	dirs.checkParents(startPath)
+	if len(dirs) == 0 {
+		dirs[startPath] = nil
+	}
+	err = dirs.Prune(toPrune)
+	if err != nil {
+		return nil, err
+	}
+	dirs.Sort()
+	return dirs, nil
+}
+
+// Create a DirTree using List
+func walkNDirTree(f fs.Fs, path string, includeAll bool, maxLevel int, listDir listDirFunc) (DirTree, error) {
+	dirs := make(DirTree)
+	fn := func(dirPath string, entries fs.DirEntries, err error) error {
+		if err == nil {
+			dirs[dirPath] = entries
+		}
+		return err
+	}
+	err := walk(f, path, includeAll, maxLevel, fn, listDir)
+	if err != nil {
+		return nil, err
+	}
+	return dirs, nil
+}
+
+// NewDirTree returns a DirTree filled with the directory listing
+// using the parameters supplied.
+//
+// If includeAll is not set it will use the filters defined.
+//
+// If maxLevel is < 0 then it will recurse indefinitely, else it will
+// only do maxLevel levels.
+//
+// This is implemented by WalkR if Config.UseRecursiveListing is true
+// and f supports it and level > 1, or WalkN otherwise.
+//
+// NB (f, path) to be replaced by fs.Dir at some point
+func NewDirTree(f fs.Fs, path string, includeAll bool, maxLevel int) (DirTree, error) {
+	if ListR := f.Features().ListR; (maxLevel < 0 || maxLevel > 1) && fs.Config.UseListR && ListR != nil {
+		return walkRDirTree(f, path, includeAll, maxLevel, ListR)
+	}
+	return walkNDirTree(f, path, includeAll, maxLevel, list.DirSorted)
+}
+
+func walkR(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listR fs.ListRFn) error {
+	dirs, err := walkRDirTree(f, path, includeAll, maxLevel, listR)
+	if err != nil {
+		return err
+	}
+	skipping := false
+	skipPrefix := ""
+	emptyDir := fs.DirEntries{}
+	for _, dirPath := range dirs.Dirs() {
+		if skipping {
+			// Skip over directories as required
+			if strings.HasPrefix(dirPath, skipPrefix) {
+				continue
+			}
+			skipping = false
+		}
+		entries := dirs[dirPath]
+		if entries == nil {
+			entries = emptyDir
+		}
+		err = fn(dirPath, entries, nil)
+		if err == ErrorSkipDir {
+			skipping = true
+			skipPrefix = dirPath
+			if skipPrefix != "" {
+				skipPrefix += "/"
+			}
+		} else if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// GetAll runs Walk getting all the results
+func GetAll(f fs.Fs, path string, includeAll bool, maxLevel int) (objs []fs.Object, dirs []fs.Directory, err error) {
+	err = Walk(f, path, includeAll, maxLevel, func(dirPath string, entries fs.DirEntries, err error) error {
+		if err != nil {
+			return err
+		}
+		for _, entry := range entries {
+			switch x := entry.(type) {
+			case fs.Object:
+				objs = append(objs, x)
+			case fs.Directory:
+				dirs = append(dirs, x)
+			}
+		}
+		return nil
+	})
+	return
+}
+
+// ListRHelper is used in the implementation of ListR to accumulate DirEntries
+type ListRHelper struct {
+	callback fs.ListRCallback
+	entries  fs.DirEntries
+}
+
+// NewListRHelper should be called from ListR with the callback passed in
+func NewListRHelper(callback fs.ListRCallback) *ListRHelper {
+	return &ListRHelper{
+		callback: callback,
+	}
+}
+
+// send sends the stored entries to the callback if there are >= max
+// entries.
+func (lh *ListRHelper) send(max int) (err error) {
+	if len(lh.entries) >= max {
+		err = lh.callback(lh.entries)
+		lh.entries = lh.entries[:0]
+	}
+	return err
+}
+
+// Add an entry to the stored entries and send them if there are more
+// than a certain amount
+func (lh *ListRHelper) Add(entry fs.DirEntry) error {
+	if entry == nil {
+		return nil
+	}
+	lh.entries = append(lh.entries, entry)
+	return lh.send(100)
+}
+
+// Flush the stored entries (if any) sending them to the callback
+func (lh *ListRHelper) Flush() error {
+	return lh.send(1)
+}