newline battles continue

vendor/github.com/ncw/rclone/fs/accounting/accounting.go

@@ -0,0 +1,391 @@
+// Package accounting providers an accounting and limiting reader
+package accounting
+
+import (
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/asyncreader"
+	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/pkg/errors"
+)
+
+// ErrorMaxTransferLimitReached is returned from Read when the max
+// transfer limit is reached.
+var ErrorMaxTransferLimitReached = fserrors.FatalError(errors.New("Max transfer limit reached as set by --max-transfer"))
+
+// Account limits and accounts for one transfer
+type Account struct {
+	// The mutex is to make sure Read() and Close() aren't called
+	// concurrently.  Unfortunately the persistent connection loop
+	// in http transport calls Read() after Do() returns on
+	// CancelRequest so this race can happen when it apparently
+	// shouldn't.
+	mu      sync.Mutex
+	in      io.Reader
+	origIn  io.ReadCloser
+	close   io.Closer
+	size    int64
+	name    string
+	statmu  sync.Mutex    // Separate mutex for stat values.
+	bytes   int64         // Total number of bytes read
+	max     int64         // if >=0 the max number of bytes to transfer
+	start   time.Time     // Start time of first read
+	lpTime  time.Time     // Time of last average measurement
+	lpBytes int           // Number of bytes read since last measurement
+	avg     float64       // Moving average of last few measurements in bytes/s
+	closed  bool          // set if the file is closed
+	exit    chan struct{} // channel that will be closed when transfer is finished
+	withBuf bool          // is using a buffered in
+}
+
+const averagePeriod = 16 // period to do exponentially weighted averages over
+
+// NewAccountSizeName makes a Account reader for an io.ReadCloser of
+// the given size and name
+func NewAccountSizeName(in io.ReadCloser, size int64, name string) *Account {
+	acc := &Account{
+		in:     in,
+		close:  in,
+		origIn: in,
+		size:   size,
+		name:   name,
+		exit:   make(chan struct{}),
+		avg:    0,
+		lpTime: time.Now(),
+		max:    int64(fs.Config.MaxTransfer),
+	}
+	go acc.averageLoop()
+	Stats.inProgress.set(acc.name, acc)
+	return acc
+}
+
+// NewAccount makes a Account reader for an object
+func NewAccount(in io.ReadCloser, obj fs.Object) *Account {
+	return NewAccountSizeName(in, obj.Size(), obj.Remote())
+}
+
+// WithBuffer - If the file is above a certain size it adds an Async reader
+func (acc *Account) WithBuffer() *Account {
+	acc.withBuf = true
+	var buffers int
+	if acc.size >= int64(fs.Config.BufferSize) || acc.size == -1 {
+		buffers = int(int64(fs.Config.BufferSize) / asyncreader.BufferSize)
+	} else {
+		buffers = int(acc.size / asyncreader.BufferSize)
+	}
+	// On big files add a buffer
+	if buffers > 0 {
+		rc, err := asyncreader.New(acc.origIn, buffers)
+		if err != nil {
+			fs.Errorf(acc.name, "Failed to make buffer: %v", err)
+		} else {
+			acc.in = rc
+			acc.close = rc
+		}
+	}
+	return acc
+}
+
+// GetReader returns the underlying io.ReadCloser under any Buffer
+func (acc *Account) GetReader() io.ReadCloser {
+	acc.mu.Lock()
+	defer acc.mu.Unlock()
+	return acc.origIn
+}
+
+// GetAsyncReader returns the current AsyncReader or nil if Account is unbuffered
+func (acc *Account) GetAsyncReader() *asyncreader.AsyncReader {
+	acc.mu.Lock()
+	defer acc.mu.Unlock()
+	if asyncIn, ok := acc.in.(*asyncreader.AsyncReader); ok {
+		return asyncIn
+	}
+	return nil
+}
+
+// StopBuffering stops the async buffer doing any more buffering
+func (acc *Account) StopBuffering() {
+	if asyncIn, ok := acc.in.(*asyncreader.AsyncReader); ok {
+		asyncIn.Abandon()
+	}
+}
+
+// UpdateReader updates the underlying io.ReadCloser stopping the
+// asynb buffer (if any) and re-adding it
+func (acc *Account) UpdateReader(in io.ReadCloser) {
+	acc.mu.Lock()
+	acc.StopBuffering()
+	acc.in = in
+	acc.close = in
+	acc.origIn = in
+	acc.WithBuffer()
+	acc.mu.Unlock()
+}
+
+// averageLoop calculates averages for the stats in the background
+func (acc *Account) averageLoop() {
+	tick := time.NewTicker(time.Second)
+	var period float64
+	defer tick.Stop()
+	for {
+		select {
+		case now := <-tick.C:
+			acc.statmu.Lock()
+			// Add average of last second.
+			elapsed := now.Sub(acc.lpTime).Seconds()
+			avg := float64(acc.lpBytes) / elapsed
+			// Soft start the moving average
+			if period < averagePeriod {
+				period++
+			}
+			acc.avg = (avg + (period-1)*acc.avg) / period
+			acc.lpBytes = 0
+			acc.lpTime = now
+			// Unlock stats
+			acc.statmu.Unlock()
+		case <-acc.exit:
+			return
+		}
+	}
+}
+
+// read bytes from the io.Reader passed in and account them
+func (acc *Account) read(in io.Reader, p []byte) (n int, err error) {
+	acc.statmu.Lock()
+	if acc.max >= 0 && Stats.GetBytes() >= acc.max {
+		acc.statmu.Unlock()
+		return 0, ErrorMaxTransferLimitReached
+	}
+	// Set start time.
+	if acc.start.IsZero() {
+		acc.start = time.Now()
+	}
+	acc.statmu.Unlock()
+
+	n, err = in.Read(p)
+
+	// Update Stats
+	acc.statmu.Lock()
+	acc.lpBytes += n
+	acc.bytes += int64(n)
+	acc.statmu.Unlock()
+
+	Stats.Bytes(int64(n))
+
+	limitBandwidth(n)
+	return
+}
+
+// Read bytes from the object - see io.Reader
+func (acc *Account) Read(p []byte) (n int, err error) {
+	acc.mu.Lock()
+	defer acc.mu.Unlock()
+	return acc.read(acc.in, p)
+}
+
+// Close the object
+func (acc *Account) Close() error {
+	acc.mu.Lock()
+	defer acc.mu.Unlock()
+	if acc.closed {
+		return nil
+	}
+	acc.closed = true
+	close(acc.exit)
+	Stats.inProgress.clear(acc.name)
+	return acc.close.Close()
+}
+
+// progress returns bytes read as well as the size.
+// Size can be <= 0 if the size is unknown.
+func (acc *Account) progress() (bytes, size int64) {
+	if acc == nil {
+		return 0, 0
+	}
+	acc.statmu.Lock()
+	bytes, size = acc.bytes, acc.size
+	acc.statmu.Unlock()
+	return bytes, size
+}
+
+// speed returns the speed of the current file transfer
+// in bytes per second, as well a an exponentially weighted moving average
+// If no read has completed yet, 0 is returned for both values.
+func (acc *Account) speed() (bps, current float64) {
+	if acc == nil {
+		return 0, 0
+	}
+	acc.statmu.Lock()
+	defer acc.statmu.Unlock()
+	if acc.bytes == 0 {
+		return 0, 0
+	}
+	// Calculate speed from first read.
+	total := float64(time.Now().Sub(acc.start)) / float64(time.Second)
+	bps = float64(acc.bytes) / total
+	current = acc.avg
+	return
+}
+
+// eta returns the ETA of the current operation,
+// rounded to full seconds.
+// If the ETA cannot be determined 'ok' returns false.
+func (acc *Account) eta() (etaDuration time.Duration, ok bool) {
+	if acc == nil {
+		return 0, false
+	}
+	acc.statmu.Lock()
+	defer acc.statmu.Unlock()
+	return eta(acc.bytes, acc.size, acc.avg)
+}
+
+// String produces stats for this file
+func (acc *Account) String() string {
+	a, b := acc.progress()
+	_, cur := acc.speed()
+	eta, etaok := acc.eta()
+	etas := "-"
+	if etaok {
+		if eta > 0 {
+			etas = fmt.Sprintf("%v", eta)
+		} else {
+			etas = "0s"
+		}
+	}
+	name := []rune(acc.name)
+	if fs.Config.StatsFileNameLength > 0 {
+		if len(name) > fs.Config.StatsFileNameLength {
+			where := len(name) - fs.Config.StatsFileNameLength
+			name = append([]rune{'.', '.', '.'}, name[where:]...)
+		}
+	}
+
+	if fs.Config.DataRateUnit == "bits" {
+		cur = cur * 8
+	}
+
+	percentageDone := 0
+	if b > 0 {
+		percentageDone = int(100 * float64(a) / float64(b))
+	}
+
+	done := fmt.Sprintf("%2d%% /%s", percentageDone, fs.SizeSuffix(b))
+
+	return fmt.Sprintf("%45s: %s, %s/s, %s",
+		string(name),
+		done,
+		fs.SizeSuffix(cur),
+		etas,
+	)
+}
+
+// RemoteStats produces stats for this file
+func (acc *Account) RemoteStats() (out map[string]interface{}) {
+	out = make(map[string]interface{})
+	a, b := acc.progress()
+	out["bytes"] = a
+	out["size"] = b
+	spd, cur := acc.speed()
+	out["speed"] = spd
+	out["speedAvg"] = cur
+
+	eta, etaok := acc.eta()
+	out["eta"] = nil
+	if etaok {
+		if eta > 0 {
+			out["eta"] = eta.Seconds()
+		} else {
+			out["eta"] = 0
+		}
+	}
+	out["name"] = acc.name
+
+	percentageDone := 0
+	if b > 0 {
+		percentageDone = int(100 * float64(a) / float64(b))
+	}
+	out["percentage"] = percentageDone
+
+	return out
+}
+
+// OldStream returns the top io.Reader
+func (acc *Account) OldStream() io.Reader {
+	acc.mu.Lock()
+	defer acc.mu.Unlock()
+	return acc.in
+}
+
+// SetStream updates the top io.Reader
+func (acc *Account) SetStream(in io.Reader) {
+	acc.mu.Lock()
+	acc.in = in
+	acc.mu.Unlock()
+}
+
+// WrapStream wraps an io Reader so it will be accounted in the same
+// way as account
+func (acc *Account) WrapStream(in io.Reader) io.Reader {
+	return &accountStream{
+		acc: acc,
+		in:  in,
+	}
+}
+
+// accountStream accounts a single io.Reader into a parent *Account
+type accountStream struct {
+	acc *Account
+	in  io.Reader
+}
+
+// OldStream return the underlying stream
+func (a *accountStream) OldStream() io.Reader {
+	return a.in
+}
+
+// SetStream set the underlying stream
+func (a *accountStream) SetStream(in io.Reader) {
+	a.in = in
+}
+
+// WrapStream wrap in in an accounter
+func (a *accountStream) WrapStream(in io.Reader) io.Reader {
+	return a.acc.WrapStream(in)
+}
+
+// Read bytes from the object - see io.Reader
+func (a *accountStream) Read(p []byte) (n int, err error) {
+	return a.acc.read(a.in, p)
+}
+
+// Accounter accounts a stream allowing the accounting to be removed and re-added
+type Accounter interface {
+	io.Reader
+	OldStream() io.Reader
+	SetStream(io.Reader)
+	WrapStream(io.Reader) io.Reader
+}
+
+// WrapFn wraps an io.Reader (for accounting purposes usually)
+type WrapFn func(io.Reader) io.Reader
+
+// UnWrap unwraps a reader returning unwrapped and wrap, a function to
+// wrap it back up again.  If `in` is an Accounter then this function
+// will take the accounting unwrapped and wrap will put it back on
+// again the new Reader passed in.
+//
+// This allows functions which wrap io.Readers to move the accounting
+// to the end of the wrapped chain of readers.  This is very important
+// if buffering is being introduced and if the Reader might be wrapped
+// again.
+func UnWrap(in io.Reader) (unwrapped io.Reader, wrap WrapFn) {
+	acc, ok := in.(Accounter)
+	if !ok {
+		return in, func(r io.Reader) io.Reader { return r }
+	}
+	return acc.OldStream(), acc.WrapStream
+}

vendor/github.com/ncw/rclone/fs/accounting/accounting_other.go

@@ -0,0 +1,10 @@
+// Accounting and limiting reader
+// Non-unix specific functions.
+
+// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris
+
+package accounting
+
+// startSignalHandler() is Unix specific and does nothing under non-Unix
+// platforms.
+func startSignalHandler() {}

vendor/github.com/ncw/rclone/fs/accounting/accounting_unix.go

@@ -0,0 +1,36 @@
+// Accounting and limiting reader
+// Unix specific functions.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd solaris
+
+package accounting
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/ncw/rclone/fs"
+)
+
+// startSignalHandler() sets a signal handler to catch SIGUSR2 and toggle throttling.
+func startSignalHandler() {
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, syscall.SIGUSR2)
+
+	go func() {
+		// This runs forever, but blocks until the signal is received.
+		for {
+			<-signals
+			tokenBucketMu.Lock()
+			bwLimitToggledOff = !bwLimitToggledOff
+			tokenBucket, prevTokenBucket = prevTokenBucket, tokenBucket
+			s := "disabled"
+			if tokenBucket != nil {
+				s = "enabled"
+			}
+			tokenBucketMu.Unlock()
+			fs.Logf(nil, "Bandwidth limit %s by user", s)
+		}
+	}()
+}

vendor/github.com/ncw/rclone/fs/accounting/inprogress.go

@@ -0,0 +1,41 @@
+package accounting
+
+import (
+	"sync"
+
+	"github.com/ncw/rclone/fs"
+)
+
+// inProgress holds a synchronized map of in progress transfers
+type inProgress struct {
+	mu sync.Mutex
+	m  map[string]*Account
+}
+
+// newInProgress makes a new inProgress object
+func newInProgress() *inProgress {
+	return &inProgress{
+		m: make(map[string]*Account, fs.Config.Transfers),
+	}
+}
+
+// set marks the name as in progress
+func (ip *inProgress) set(name string, acc *Account) {
+	ip.mu.Lock()
+	defer ip.mu.Unlock()
+	ip.m[name] = acc
+}
+
+// clear marks the name as no longer in progress
+func (ip *inProgress) clear(name string) {
+	ip.mu.Lock()
+	defer ip.mu.Unlock()
+	delete(ip.m, name)
+}
+
+// get gets the account for name, of nil if not found
+func (ip *inProgress) get(name string) *Account {
+	ip.mu.Lock()
+	defer ip.mu.Unlock()
+	return ip.m[name]
+}

vendor/github.com/ncw/rclone/fs/accounting/stats.go

@@ -0,0 +1,406 @@
+package accounting
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/rc"
+)
+
+var (
+	// Stats is global statistics counter
+	Stats = NewStats()
+)
+
+func init() {
+	// Set the function pointer up in fs
+	fs.CountError = Stats.Error
+
+	rc.Add(rc.Call{
+		Path:  "core/stats",
+		Fn:    Stats.RemoteStats,
+		Title: "Returns stats about current transfers.",
+		Help: `
+This returns all available stats
+
+	rclone rc core/stats
+
+Returns the following values:
+
+` + "```" + `
+{
+	"speed": average speed in bytes/sec since start of the process,
+	"bytes": total transferred bytes since the start of the process,
+	"errors": number of errors,
+	"checks": number of checked files,
+	"transfers": number of transferred files,
+	"deletes" : number of deleted files,
+	"elapsedTime": time in seconds since the start of the process,
+	"lastError": last occurred error,
+	"transferring": an array of currently active file transfers:
+		[
+			{
+				"bytes": total transferred bytes for this file,
+				"eta": estimated time in seconds until file transfer completion
+				"name": name of the file,
+				"percentage": progress of the file transfer in percent,
+				"speed": speed in bytes/sec,
+				"speedAvg": speed in bytes/sec as an exponentially weighted moving average,
+				"size": size of the file in bytes
+			}
+		],
+	"checking": an array of names of currently active file checks
+		[]
+}
+` + "```" + `
+Values for "transferring", "checking" and "lastError" are only assigned if data is available.
+The value for "eta" is null if an eta cannot be determined.
+`,
+	})
+}
+
+// StatsInfo accounts all transfers
+type StatsInfo struct {
+	mu                sync.RWMutex
+	bytes             int64
+	errors            int64
+	lastError         error
+	checks            int64
+	checking          *stringSet
+	checkQueue        int
+	checkQueueSize    int64
+	transfers         int64
+	transferring      *stringSet
+	transferQueue     int
+	transferQueueSize int64
+	renameQueue       int
+	renameQueueSize   int64
+	deletes           int64
+	start             time.Time
+	inProgress        *inProgress
+}
+
+// NewStats cretates an initialised StatsInfo
+func NewStats() *StatsInfo {
+	return &StatsInfo{
+		checking:     newStringSet(fs.Config.Checkers),
+		transferring: newStringSet(fs.Config.Transfers),
+		start:        time.Now(),
+		inProgress:   newInProgress(),
+	}
+}
+
+// RemoteStats returns stats for rc
+func (s *StatsInfo) RemoteStats(in rc.Params) (out rc.Params, err error) {
+	out = make(rc.Params)
+	s.mu.RLock()
+	dt := time.Now().Sub(s.start)
+	dtSeconds := dt.Seconds()
+	speed := 0.0
+	if dt > 0 {
+		speed = float64(s.bytes) / dtSeconds
+	}
+	out["speed"] = speed
+	out["bytes"] = s.bytes
+	out["errors"] = s.errors
+	out["checks"] = s.checks
+	out["transfers"] = s.transfers
+	out["deletes"] = s.deletes
+	out["elapsedTime"] = dtSeconds
+	s.mu.RUnlock()
+	if !s.checking.empty() {
+		var c []string
+		s.checking.mu.RLock()
+		defer s.checking.mu.RUnlock()
+		for name := range s.checking.items {
+			c = append(c, name)
+		}
+		out["checking"] = c
+	}
+	if !s.transferring.empty() {
+		var t []interface{}
+		s.transferring.mu.RLock()
+		defer s.transferring.mu.RUnlock()
+		for name := range s.transferring.items {
+			if acc := s.inProgress.get(name); acc != nil {
+				t = append(t, acc.RemoteStats())
+			} else {
+				t = append(t, name)
+			}
+		}
+		out["transferring"] = t
+	}
+	if s.errors > 0 {
+		out["lastError"] = s.lastError
+	}
+	return out, nil
+}
+
+// eta returns the ETA of the current operation,
+// rounded to full seconds.
+// If the ETA cannot be determined 'ok' returns false.
+func eta(size, total int64, rate float64) (eta time.Duration, ok bool) {
+	if total <= 0 || size < 0 || rate <= 0 {
+		return 0, false
+	}
+	remaining := total - size
+	if remaining < 0 {
+		return 0, false
+	}
+	seconds := float64(remaining) / rate
+	return time.Second * time.Duration(seconds), true
+}
+
+// etaString returns the ETA of the current operation,
+// rounded to full seconds.
+// If the ETA cannot be determined it returns "-"
+func etaString(done, total int64, rate float64) string {
+	d, ok := eta(done, total, rate)
+	if !ok {
+		return "-"
+	}
+	return d.String()
+}
+
+// percent returns a/b as a percentage rounded to the nearest integer
+// as a string
+//
+// if the percentage is invalid it returns "-"
+func percent(a int64, b int64) string {
+	if a < 0 || b <= 0 {
+		return "-"
+	}
+	return fmt.Sprintf("%d%%", int(float64(a)*100/float64(b)+0.5))
+}
+
+// String convert the StatsInfo to a string for printing
+func (s *StatsInfo) String() string {
+	// checking and transferring have their own locking so read
+	// here before lock to prevent deadlock on GetBytes
+	transferring, checking := s.transferring.count(), s.checking.count()
+	transferringBytesDone, transferringBytesTotal := s.transferring.progress()
+
+	s.mu.RLock()
+
+	dt := time.Now().Sub(s.start)
+	dtSeconds := dt.Seconds()
+	speed := 0.0
+	if dt > 0 {
+		speed = float64(s.bytes) / dtSeconds
+	}
+	dtRounded := dt - (dt % (time.Second / 10))
+
+	if fs.Config.DataRateUnit == "bits" {
+		speed = speed * 8
+	}
+
+	var (
+		totalChecks   = int64(s.checkQueue) + s.checks + int64(checking)
+		totalTransfer = int64(s.transferQueue) + s.transfers + int64(transferring)
+		// note that s.bytes already includes transferringBytesDone so
+		// we take it off here to avoid double counting
+		totalSize    = s.transferQueueSize + s.bytes + transferringBytesTotal - transferringBytesDone
+		currentSize  = s.bytes
+		buf          = &bytes.Buffer{}
+		xfrchkString = ""
+	)
+
+	if !fs.Config.StatsOneLine {
+		_, _ = fmt.Fprintf(buf, "\nTransferred:   	")
+	} else {
+		xfrchk := []string{}
+		if totalTransfer > 0 && s.transferQueue > 0 {
+			xfrchk = append(xfrchk, fmt.Sprintf("xfr#%d/%d", s.transfers, totalTransfer))
+		}
+		if totalChecks > 0 && s.checkQueue > 0 {
+			xfrchk = append(xfrchk, fmt.Sprintf("chk#%d/%d", s.checks, totalChecks))
+		}
+		if len(xfrchk) > 0 {
+			xfrchkString = fmt.Sprintf(" (%s)", strings.Join(xfrchk, ", "))
+		}
+	}
+
+	_, _ = fmt.Fprintf(buf, "%10s / %s, %s, %s, ETA %s%s",
+		fs.SizeSuffix(s.bytes),
+		fs.SizeSuffix(totalSize).Unit("Bytes"),
+		percent(s.bytes, totalSize),
+		fs.SizeSuffix(speed).Unit(strings.Title(fs.Config.DataRateUnit)+"/s"),
+		etaString(currentSize, totalSize, speed),
+		xfrchkString,
+	)
+
+	if !fs.Config.StatsOneLine {
+		_, _ = fmt.Fprintf(buf, `
+Errors:        %10d
+Checks:        %10d / %d, %s
+Transferred:   %10d / %d, %s
+Elapsed time:  %10v
+`,
+			s.errors,
+			s.checks, totalChecks, percent(s.checks, totalChecks),
+			s.transfers, totalTransfer, percent(s.transfers, totalTransfer),
+			dtRounded)
+	}
+
+	// checking and transferring have their own locking so unlock
+	// here to prevent deadlock on GetBytes
+	s.mu.RUnlock()
+
+	// Add per transfer stats if required
+	if !fs.Config.StatsOneLine {
+		if !s.checking.empty() {
+			_, _ = fmt.Fprintf(buf, "Checking:\n%s\n", s.checking)
+		}
+		if !s.transferring.empty() {
+			_, _ = fmt.Fprintf(buf, "Transferring:\n%s\n", s.transferring)
+		}
+	}
+
+	return buf.String()
+}
+
+// Log outputs the StatsInfo to the log
+func (s *StatsInfo) Log() {
+	fs.LogLevelPrintf(fs.Config.StatsLogLevel, nil, "%v\n", s)
+}
+
+// Bytes updates the stats for bytes bytes
+func (s *StatsInfo) Bytes(bytes int64) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.bytes += bytes
+}
+
+// GetBytes returns the number of bytes transferred so far
+func (s *StatsInfo) GetBytes() int64 {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.bytes
+}
+
+// Errors updates the stats for errors
+func (s *StatsInfo) Errors(errors int64) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.errors += errors
+}
+
+// GetErrors reads the number of errors
+func (s *StatsInfo) GetErrors() int64 {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.errors
+}
+
+// GetLastError returns the lastError
+func (s *StatsInfo) GetLastError() error {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.lastError
+}
+
+// Deletes updates the stats for deletes
+func (s *StatsInfo) Deletes(deletes int64) int64 {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.deletes += deletes
+	return s.deletes
+}
+
+// ResetCounters sets the counters (bytes, checks, errors, transfers) to 0
+func (s *StatsInfo) ResetCounters() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.bytes = 0
+	s.errors = 0
+	s.checks = 0
+	s.transfers = 0
+	s.deletes = 0
+}
+
+// ResetErrors sets the errors count to 0
+func (s *StatsInfo) ResetErrors() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.errors = 0
+}
+
+// Errored returns whether there have been any errors
+func (s *StatsInfo) Errored() bool {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.errors != 0
+}
+
+// Error adds a single error into the stats and assigns lastError
+func (s *StatsInfo) Error(err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.errors++
+	s.lastError = err
+}
+
+// Checking adds a check into the stats
+func (s *StatsInfo) Checking(remote string) {
+	s.checking.add(remote)
+}
+
+// DoneChecking removes a check from the stats
+func (s *StatsInfo) DoneChecking(remote string) {
+	s.checking.del(remote)
+	s.mu.Lock()
+	s.checks++
+	s.mu.Unlock()
+}
+
+// GetTransfers reads the number of transfers
+func (s *StatsInfo) GetTransfers() int64 {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.transfers
+}
+
+// Transferring adds a transfer into the stats
+func (s *StatsInfo) Transferring(remote string) {
+	s.transferring.add(remote)
+}
+
+// DoneTransferring removes a transfer from the stats
+//
+// if ok is true then it increments the transfers count
+func (s *StatsInfo) DoneTransferring(remote string, ok bool) {
+	s.transferring.del(remote)
+	if ok {
+		s.mu.Lock()
+		s.transfers++
+		s.mu.Unlock()
+	}
+}
+
+// SetCheckQueue sets the number of queued checks
+func (s *StatsInfo) SetCheckQueue(n int, size int64) {
+	s.mu.Lock()
+	s.checkQueue = n
+	s.checkQueueSize = size
+	s.mu.Unlock()
+}
+
+// SetTransferQueue sets the number of queued transfers
+func (s *StatsInfo) SetTransferQueue(n int, size int64) {
+	s.mu.Lock()
+	s.transferQueue = n
+	s.transferQueueSize = size
+	s.mu.Unlock()
+}
+
+// SetRenameQueue sets the number of queued transfers
+func (s *StatsInfo) SetRenameQueue(n int, size int64) {
+	s.mu.Lock()
+	s.renameQueue = n
+	s.renameQueueSize = size
+	s.mu.Unlock()
+}

vendor/github.com/ncw/rclone/fs/accounting/stringset.go

@@ -0,0 +1,88 @@
+package accounting
+
+import (
+	"sort"
+	"strings"
+	"sync"
+)
+
+// stringSet holds a set of strings
+type stringSet struct {
+	mu    sync.RWMutex
+	items map[string]struct{}
+}
+
+// newStringSet creates a new empty string set of capacity size
+func newStringSet(size int) *stringSet {
+	return &stringSet{
+		items: make(map[string]struct{}, size),
+	}
+}
+
+// add adds remote to the set
+func (ss *stringSet) add(remote string) {
+	ss.mu.Lock()
+	ss.items[remote] = struct{}{}
+	ss.mu.Unlock()
+}
+
+// del removes remote from the set
+func (ss *stringSet) del(remote string) {
+	ss.mu.Lock()
+	delete(ss.items, remote)
+	ss.mu.Unlock()
+}
+
+// empty returns whether the set has any items
+func (ss *stringSet) empty() bool {
+	ss.mu.RLock()
+	defer ss.mu.RUnlock()
+	return len(ss.items) == 0
+}
+
+// count returns the number of items in the set
+func (ss *stringSet) count() int {
+	ss.mu.RLock()
+	defer ss.mu.RUnlock()
+	return len(ss.items)
+}
+
+// Strings returns all the strings in the stringSet
+func (ss *stringSet) Strings() []string {
+	ss.mu.RLock()
+	defer ss.mu.RUnlock()
+	strings := make([]string, 0, len(ss.items))
+	for name := range ss.items {
+		var out string
+		if acc := Stats.inProgress.get(name); acc != nil {
+			out = acc.String()
+		} else {
+			out = name
+		}
+		strings = append(strings, " * "+out)
+	}
+	sorted := sort.StringSlice(strings)
+	sorted.Sort()
+	return sorted
+}
+
+// String returns all the file names in the stringSet joined by newline
+func (ss *stringSet) String() string {
+	return strings.Join(ss.Strings(), "\n")
+}
+
+// progress returns total bytes read as well as the size.
+func (ss *stringSet) progress() (totalBytes, totalSize int64) {
+	ss.mu.RLock()
+	defer ss.mu.RUnlock()
+	for name := range ss.items {
+		if acc := Stats.inProgress.get(name); acc != nil {
+			bytes, size := acc.progress()
+			if size >= 0 && bytes >= 0 {
+				totalBytes += bytes
+				totalSize += size
+			}
+		}
+	}
+	return totalBytes, totalSize
+}

vendor/github.com/ncw/rclone/fs/accounting/token_bucket.go

@@ -0,0 +1,169 @@
+package accounting
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/rc"
+	"github.com/pkg/errors"
+	"golang.org/x/time/rate"
+)
+
+// Globals
+var (
+	tokenBucketMu     sync.Mutex // protects the token bucket variables
+	tokenBucket       *rate.Limiter
+	prevTokenBucket   = tokenBucket
+	bwLimitToggledOff = false
+	currLimitMu       sync.Mutex // protects changes to the timeslot
+	currLimit         fs.BwTimeSlot
+)
+
+const maxBurstSize = 4 * 1024 * 1024 // must be bigger than the biggest request
+
+// make a new empty token bucket with the bandwidth given
+func newTokenBucket(bandwidth fs.SizeSuffix) *rate.Limiter {
+	newTokenBucket := rate.NewLimiter(rate.Limit(bandwidth), maxBurstSize)
+	// empty the bucket
+	err := newTokenBucket.WaitN(context.Background(), maxBurstSize)
+	if err != nil {
+		fs.Errorf(nil, "Failed to empty token bucket: %v", err)
+	}
+	return newTokenBucket
+}
+
+// StartTokenBucket starts the token bucket if necessary
+func StartTokenBucket() {
+	currLimitMu.Lock()
+	currLimit := fs.Config.BwLimit.LimitAt(time.Now())
+	currLimitMu.Unlock()
+
+	if currLimit.Bandwidth > 0 {
+		tokenBucket = newTokenBucket(currLimit.Bandwidth)
+		fs.Infof(nil, "Starting bandwidth limiter at %vBytes/s", &currLimit.Bandwidth)
+
+		// Start the SIGUSR2 signal handler to toggle bandwidth.
+		// This function does nothing in windows systems.
+		startSignalHandler()
+	}
+}
+
+// StartTokenTicker creates a ticker to update the bandwidth limiter every minute.
+func StartTokenTicker() {
+	// If the timetable has a single entry or was not specified, we don't need
+	// a ticker to update the bandwidth.
+	if len(fs.Config.BwLimit) <= 1 {
+		return
+	}
+
+	ticker := time.NewTicker(time.Minute)
+	go func() {
+		for range ticker.C {
+			limitNow := fs.Config.BwLimit.LimitAt(time.Now())
+			currLimitMu.Lock()
+
+			if currLimit.Bandwidth != limitNow.Bandwidth {
+				tokenBucketMu.Lock()
+
+				// If bwlimit is toggled off, the change should only
+				// become active on the next toggle, which causes
+				// an exchange of tokenBucket <-> prevTokenBucket
+				var targetBucket **rate.Limiter
+				if bwLimitToggledOff {
+					targetBucket = &prevTokenBucket
+				} else {
+					targetBucket = &tokenBucket
+				}
+
+				// Set new bandwidth. If unlimited, set tokenbucket to nil.
+				if limitNow.Bandwidth > 0 {
+					*targetBucket = newTokenBucket(limitNow.Bandwidth)
+					if bwLimitToggledOff {
+						fs.Logf(nil, "Scheduled bandwidth change. "+
+							"Limit will be set to %vBytes/s when toggled on again.", &limitNow.Bandwidth)
+					} else {
+						fs.Logf(nil, "Scheduled bandwidth change. Limit set to %vBytes/s", &limitNow.Bandwidth)
+					}
+				} else {
+					*targetBucket = nil
+					fs.Logf(nil, "Scheduled bandwidth change. Bandwidth limits disabled")
+				}
+
+				currLimit = limitNow
+				tokenBucketMu.Unlock()
+			}
+			currLimitMu.Unlock()
+		}
+	}()
+}
+
+// limitBandwith sleeps for the correct amount of time for the passage
+// of n bytes according to the current bandwidth limit
+func limitBandwidth(n int) {
+	tokenBucketMu.Lock()
+
+	// Limit the transfer speed if required
+	if tokenBucket != nil {
+		err := tokenBucket.WaitN(context.Background(), n)
+		if err != nil {
+			fs.Errorf(nil, "Token bucket error: %v", err)
+		}
+	}
+
+	tokenBucketMu.Unlock()
+}
+
+// SetBwLimit sets the current bandwidth limit
+func SetBwLimit(bandwidth fs.SizeSuffix) {
+	tokenBucketMu.Lock()
+	defer tokenBucketMu.Unlock()
+	if bandwidth > 0 {
+		tokenBucket = newTokenBucket(bandwidth)
+		fs.Logf(nil, "Bandwidth limit set to %v", bandwidth)
+	} else {
+		tokenBucket = nil
+		fs.Logf(nil, "Bandwidth limit reset to unlimited")
+	}
+}
+
+// Remote control for the token bucket
+func init() {
+	rc.Add(rc.Call{
+		Path: "core/bwlimit",
+		Fn: func(in rc.Params) (out rc.Params, err error) {
+			ibwlimit, ok := in["rate"]
+			if !ok {
+				return out, errors.Errorf("parameter rate not found")
+			}
+			bwlimit, ok := ibwlimit.(string)
+			if !ok {
+				return out, errors.Errorf("value must be string rate=%v", ibwlimit)
+			}
+			var bws fs.BwTimetable
+			err = bws.Set(bwlimit)
+			if err != nil {
+				return out, errors.Wrap(err, "bad bwlimit")
+			}
+			if len(bws) != 1 {
+				return out, errors.New("need exactly 1 bandwidth setting")
+			}
+			bw := bws[0]
+			SetBwLimit(bw.Bandwidth)
+			return rc.Params{"rate": bw.Bandwidth.String()}, nil
+		},
+		Title: "Set the bandwidth limit.",
+		Help: `
+This sets the bandwidth limit to that passed in.
+
+Eg
+
+    rclone rc core/bwlimit rate=1M
+    rclone rc core/bwlimit rate=off
+
+The format of the parameter is exactly the same as passed to --bwlimit
+except only one bandwidth may be specified.
+`,
+	})
+}