overdue

.rclone_repo/lib/pacer/pacer.go

@@ -0,0 +1,368 @@
+// Package pacer makes pacing and retrying API calls easy
+package pacer
+
+import (
+	"math/rand"
+	"sync"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/fserrors"
+)
+
+// Pacer state
+type Pacer struct {
+	mu                 sync.Mutex    // Protecting read/writes
+	minSleep           time.Duration // minimum sleep time
+	maxSleep           time.Duration // maximum sleep time
+	decayConstant      uint          // decay constant
+	attackConstant     uint          // attack constant
+	pacer              chan struct{} // To pace the operations
+	sleepTime          time.Duration // Time to sleep for each transaction
+	retries            int           // Max number of retries
+	maxConnections     int           // Maximum number of concurrent connections
+	connTokens         chan struct{} // Connection tokens
+	calculatePace      func(bool)    // switchable pacing algorithm - call with mu held
+	consecutiveRetries int           // number of consecutive retries
+}
+
+// Type is for selecting different pacing algorithms
+type Type int
+
+const (
+	// DefaultPacer is a truncated exponential attack and decay.
+	//
+	// On retries the sleep time is doubled, on non errors then
+	// sleeptime decays according to the decay constant as set
+	// with SetDecayConstant.
+	//
+	// The sleep never goes below that set with SetMinSleep or
+	// above that set with SetMaxSleep.
+	DefaultPacer = Type(iota)
+
+	// AmazonCloudDrivePacer is a specialised pacer for Amazon Drive
+	//
+	// It implements a truncated exponential backoff strategy with
+	// randomization.  Normally operations are paced at the
+	// interval set with SetMinSleep.  On errors the sleep timer
+	// is set to 0..2**retries seconds.
+	//
+	// See https://developer.amazon.com/public/apis/experience/cloud-drive/content/restful-api-best-practices
+	AmazonCloudDrivePacer
+
+	// GoogleDrivePacer is a specialised pacer for Google Drive
+	//
+	// It implements a truncated exponential backoff strategy with
+	// randomization.  Normally operations are paced at the
+	// interval set with SetMinSleep.  On errors the sleep timer
+	// is set to (2 ^ n) + random_number_milliseconds seconds
+	//
+	// See https://developers.google.com/drive/v2/web/handle-errors#exponential-backoff
+	GoogleDrivePacer
+)
+
+// Paced is a function which is called by the Call and CallNoRetry
+// methods.  It should return a boolean, true if it would like to be
+// retried, and an error.  This error may be returned or returned
+// wrapped in a RetryError.
+type Paced func() (bool, error)
+
+// New returns a Pacer with sensible defaults
+func New() *Pacer {
+	p := &Pacer{
+		minSleep:       10 * time.Millisecond,
+		maxSleep:       2 * time.Second,
+		decayConstant:  2,
+		attackConstant: 1,
+		retries:        fs.Config.LowLevelRetries,
+		pacer:          make(chan struct{}, 1),
+	}
+	p.sleepTime = p.minSleep
+	p.SetPacer(DefaultPacer)
+	p.SetMaxConnections(fs.Config.Checkers + fs.Config.Transfers)
+
+	// Put the first pacing token in
+	p.pacer <- struct{}{}
+
+	return p
+}
+
+// SetSleep sets the current sleep time
+func (p *Pacer) SetSleep(t time.Duration) *Pacer {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.sleepTime = t
+	return p
+}
+
+// GetSleep gets the current sleep time
+func (p *Pacer) GetSleep() time.Duration {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.sleepTime
+}
+
+// SetMinSleep sets the minimum sleep time for the pacer
+func (p *Pacer) SetMinSleep(t time.Duration) *Pacer {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.minSleep = t
+	p.sleepTime = p.minSleep
+	return p
+}
+
+// SetMaxSleep sets the maximum sleep time for the pacer
+func (p *Pacer) SetMaxSleep(t time.Duration) *Pacer {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.maxSleep = t
+	p.sleepTime = p.minSleep
+	return p
+}
+
+// SetMaxConnections sets the maximum number of concurrent connections.
+// Setting the value to 0 will allow unlimited number of connections.
+// Should not be changed once you have started calling the pacer.
+// By default this will be set to fs.Config.Checkers.
+func (p *Pacer) SetMaxConnections(n int) *Pacer {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.maxConnections = n
+	if n <= 0 {
+		p.connTokens = nil
+	} else {
+		p.connTokens = make(chan struct{}, n)
+		for i := 0; i < n; i++ {
+			p.connTokens <- struct{}{}
+		}
+	}
+	return p
+}
+
+// SetDecayConstant sets the decay constant for the pacer
+//
+// This is the speed the time falls back to the minimum after errors
+// have occurred.
+//
+// bigger for slower decay, exponential. 1 is halve, 0 is go straight to minimum
+func (p *Pacer) SetDecayConstant(decay uint) *Pacer {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.decayConstant = decay
+	return p
+}
+
+// SetAttackConstant sets the attack constant for the pacer
+//
+// This is the speed the time grows from the minimum after errors have
+// occurred.
+//
+// bigger for slower attack, 1 is double, 0 is go straight to maximum
+func (p *Pacer) SetAttackConstant(attack uint) *Pacer {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.attackConstant = attack
+	return p
+}
+
+// SetRetries sets the max number of tries for Call
+func (p *Pacer) SetRetries(retries int) *Pacer {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.retries = retries
+	return p
+}
+
+// SetPacer sets the pacing algorithm
+//
+// It will choose the default algorithm if an incorrect value is
+// passed in.
+func (p *Pacer) SetPacer(t Type) *Pacer {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	switch t {
+	case AmazonCloudDrivePacer:
+		p.calculatePace = p.acdPacer
+	case GoogleDrivePacer:
+		p.calculatePace = p.drivePacer
+	default:
+		p.calculatePace = p.defaultPacer
+	}
+	return p
+}
+
+// Start a call to the API
+//
+// This must be called as a pair with endCall
+//
+// This waits for the pacer token
+func (p *Pacer) beginCall() {
+	// pacer starts with a token in and whenever we take one out
+	// XXX ms later we put another in.  We could do this with a
+	// Ticker more accurately, but then we'd have to work out how
+	// not to run it when it wasn't needed
+	<-p.pacer
+	if p.maxConnections > 0 {
+		<-p.connTokens
+	}
+
+	p.mu.Lock()
+	// Restart the timer
+	go func(t time.Duration) {
+		// fs.Debugf(f, "New sleep for %v at %v", t, time.Now())
+		time.Sleep(t)
+		p.pacer <- struct{}{}
+	}(p.sleepTime)
+	p.mu.Unlock()
+}
+
+// exponentialImplementation implements a exponentialImplementation up
+// and down pacing algorithm
+//
+// See the description for DefaultPacer
+//
+// This should calculate a new sleepTime.  It takes a boolean as to
+// whether the operation should be retried or not.
+//
+// Call with p.mu held
+func (p *Pacer) defaultPacer(retry bool) {
+	oldSleepTime := p.sleepTime
+	if retry {
+		if p.attackConstant == 0 {
+			p.sleepTime = p.maxSleep
+		} else {
+			p.sleepTime = (p.sleepTime << p.attackConstant) / ((1 << p.attackConstant) - 1)
+		}
+		if p.sleepTime > p.maxSleep {
+			p.sleepTime = p.maxSleep
+		}
+		if p.sleepTime != oldSleepTime {
+			fs.Debugf("pacer", "Rate limited, increasing sleep to %v", p.sleepTime)
+		}
+	} else {
+		p.sleepTime = (p.sleepTime<<p.decayConstant - p.sleepTime) >> p.decayConstant
+		if p.sleepTime < p.minSleep {
+			p.sleepTime = p.minSleep
+		}
+		if p.sleepTime != oldSleepTime {
+			fs.Debugf("pacer", "Reducing sleep to %v", p.sleepTime)
+		}
+	}
+}
+
+// acdPacer implements a truncated exponential backoff
+// strategy with randomization for Amazon Drive
+//
+// See the description for AmazonCloudDrivePacer
+//
+// This should calculate a new sleepTime.  It takes a boolean as to
+// whether the operation should be retried or not.
+//
+// Call with p.mu held
+func (p *Pacer) acdPacer(retry bool) {
+	consecutiveRetries := p.consecutiveRetries
+	if consecutiveRetries == 0 {
+		if p.sleepTime != p.minSleep {
+			p.sleepTime = p.minSleep
+			fs.Debugf("pacer", "Resetting sleep to minimum %v on success", p.sleepTime)
+		}
+	} else {
+		if consecutiveRetries > 9 {
+			consecutiveRetries = 9
+		}
+		// consecutiveRetries starts at 1 so
+		// maxSleep is 2**(consecutiveRetries-1) seconds
+		maxSleep := time.Second << uint(consecutiveRetries-1)
+		// actual sleep is random from 0..maxSleep
+		p.sleepTime = time.Duration(rand.Int63n(int64(maxSleep)))
+		if p.sleepTime < p.minSleep {
+			p.sleepTime = p.minSleep
+		}
+		fs.Debugf("pacer", "Rate limited, sleeping for %v (%d consecutive low level retries)", p.sleepTime, p.consecutiveRetries)
+	}
+}
+
+// drivePacer implements a truncated exponential backoff strategy with
+// randomization for Google Drive
+//
+// See the description for GoogleDrivePacer
+//
+// This should calculate a new sleepTime.  It takes a boolean as to
+// whether the operation should be retried or not.
+//
+// Call with p.mu held
+func (p *Pacer) drivePacer(retry bool) {
+	consecutiveRetries := p.consecutiveRetries
+	if consecutiveRetries == 0 {
+		if p.sleepTime != p.minSleep {
+			p.sleepTime = p.minSleep
+			fs.Debugf("pacer", "Resetting sleep to minimum %v on success", p.sleepTime)
+		}
+	} else {
+		if consecutiveRetries > 5 {
+			consecutiveRetries = 5
+		}
+		// consecutiveRetries starts at 1 so go from 1,2,3,4,5,5 => 1,2,4,8,16,16
+		// maxSleep is 2**(consecutiveRetries-1) seconds + random milliseconds
+		p.sleepTime = time.Second<<uint(consecutiveRetries-1) + time.Duration(rand.Int63n(int64(time.Second)))
+		fs.Debugf("pacer", "Rate limited, sleeping for %v (%d consecutive low level retries)", p.sleepTime, p.consecutiveRetries)
+	}
+}
+
+// endCall implements the pacing algorithm
+//
+// This should calculate a new sleepTime.  It takes a boolean as to
+// whether the operation should be retried or not.
+func (p *Pacer) endCall(retry bool) {
+	if p.maxConnections > 0 {
+		p.connTokens <- struct{}{}
+	}
+	p.mu.Lock()
+	if retry {
+		p.consecutiveRetries++
+	} else {
+		p.consecutiveRetries = 0
+	}
+	p.calculatePace(retry)
+	p.mu.Unlock()
+}
+
+// call implements Call but with settable retries
+func (p *Pacer) call(fn Paced, retries int) (err error) {
+	var retry bool
+	for i := 1; i <= retries; i++ {
+		p.beginCall()
+		retry, err = fn()
+		p.endCall(retry)
+		if !retry {
+			break
+		}
+		fs.Debugf("pacer", "low level retry %d/%d (error %v)", i, retries, err)
+	}
+	if retry {
+		err = fserrors.RetryError(err)
+	}
+	return err
+}
+
+// Call paces the remote operations to not exceed the limits and retry
+// on rate limit exceeded
+//
+// This calls fn, expecting it to return a retry flag and an
+// error. This error may be returned wrapped in a RetryError if the
+// number of retries is exceeded.
+func (p *Pacer) Call(fn Paced) (err error) {
+	p.mu.Lock()
+	retries := p.retries
+	p.mu.Unlock()
+	return p.call(fn, retries)
+}
+
+// CallNoRetry paces the remote operations to not exceed the limits
+// and return a retry error on rate limit exceeded
+//
+// This calls fn and wraps the output in a RetryError if it would like
+// it to be retried
+func (p *Pacer) CallNoRetry(fn Paced) error {
+	return p.call(fn, 1)
+}

.rclone_repo/lib/pacer/pacer_test.go

@@ -0,0 +1,437 @@
+package pacer
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/pkg/errors"
+)
+
+func TestNew(t *testing.T) {
+	const expectedRetries = 7
+	fs.Config.LowLevelRetries = expectedRetries
+	p := New()
+	if p.minSleep != 10*time.Millisecond {
+		t.Errorf("minSleep")
+	}
+	if p.maxSleep != 2*time.Second {
+		t.Errorf("maxSleep")
+	}
+	if p.sleepTime != p.minSleep {
+		t.Errorf("sleepTime")
+	}
+	if p.retries != expectedRetries {
+		t.Errorf("retries want %v got %v", expectedRetries, p.retries)
+	}
+	if p.decayConstant != 2 {
+		t.Errorf("decayConstant")
+	}
+	if p.attackConstant != 1 {
+		t.Errorf("attackConstant")
+	}
+	if cap(p.pacer) != 1 {
+		t.Errorf("pacer 1")
+	}
+	if len(p.pacer) != 1 {
+		t.Errorf("pacer 2")
+	}
+	if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.defaultPacer) {
+		t.Errorf("calculatePace")
+	}
+	if p.maxConnections != fs.Config.Checkers+fs.Config.Transfers {
+		t.Errorf("maxConnections")
+	}
+	if cap(p.connTokens) != fs.Config.Checkers+fs.Config.Transfers {
+		t.Errorf("connTokens")
+	}
+	if p.consecutiveRetries != 0 {
+		t.Errorf("consecutiveRetries")
+	}
+}
+
+func TestSetSleep(t *testing.T) {
+	p := New().SetSleep(2 * time.Millisecond)
+	if p.sleepTime != 2*time.Millisecond {
+		t.Errorf("didn't set")
+	}
+}
+
+func TestGetSleep(t *testing.T) {
+	p := New().SetSleep(2 * time.Millisecond)
+	if p.GetSleep() != 2*time.Millisecond {
+		t.Errorf("didn't get")
+	}
+}
+
+func TestSetMinSleep(t *testing.T) {
+	p := New().SetMinSleep(1 * time.Millisecond)
+	if p.minSleep != 1*time.Millisecond {
+		t.Errorf("didn't set")
+	}
+}
+
+func TestSetMaxSleep(t *testing.T) {
+	p := New().SetMaxSleep(100 * time.Second)
+	if p.maxSleep != 100*time.Second {
+		t.Errorf("didn't set")
+	}
+}
+
+func TestMaxConnections(t *testing.T) {
+	p := New().SetMaxConnections(20)
+	if p.maxConnections != 20 {
+		t.Errorf("maxConnections")
+	}
+	if cap(p.connTokens) != 20 {
+		t.Errorf("connTokens")
+	}
+	p.SetMaxConnections(0)
+	if p.maxConnections != 0 {
+		t.Errorf("maxConnections is not 0")
+	}
+	if p.connTokens != nil {
+		t.Errorf("connTokens is not nil")
+	}
+}
+
+func TestSetDecayConstant(t *testing.T) {
+	p := New().SetDecayConstant(17)
+	if p.decayConstant != 17 {
+		t.Errorf("didn't set")
+	}
+}
+
+func TestDecay(t *testing.T) {
+	p := New().SetMinSleep(time.Microsecond).SetPacer(DefaultPacer).SetMaxSleep(time.Second)
+	for _, test := range []struct {
+		in             time.Duration
+		attackConstant uint
+		want           time.Duration
+	}{
+		{8 * time.Millisecond, 1, 4 * time.Millisecond},
+		{1 * time.Millisecond, 0, time.Microsecond},
+		{1 * time.Millisecond, 2, (3 * time.Millisecond) / 4},
+		{1 * time.Millisecond, 3, (7 * time.Millisecond) / 8},
+	} {
+		p.sleepTime = test.in
+		p.SetDecayConstant(test.attackConstant)
+		p.defaultPacer(false)
+		got := p.sleepTime
+		if got != test.want {
+			t.Errorf("bad sleep want %v got %v", test.want, got)
+		}
+	}
+}
+
+func TestSetAttackConstant(t *testing.T) {
+	p := New().SetAttackConstant(19)
+	if p.attackConstant != 19 {
+		t.Errorf("didn't set")
+	}
+}
+
+func TestAttack(t *testing.T) {
+	p := New().SetMinSleep(time.Microsecond).SetPacer(DefaultPacer).SetMaxSleep(time.Second)
+	for _, test := range []struct {
+		in             time.Duration
+		attackConstant uint
+		want           time.Duration
+	}{
+		{1 * time.Millisecond, 1, 2 * time.Millisecond},
+		{1 * time.Millisecond, 0, time.Second},
+		{1 * time.Millisecond, 2, (4 * time.Millisecond) / 3},
+		{1 * time.Millisecond, 3, (8 * time.Millisecond) / 7},
+	} {
+		p.sleepTime = test.in
+		p.SetAttackConstant(test.attackConstant)
+		p.defaultPacer(true)
+		got := p.sleepTime
+		if got != test.want {
+			t.Errorf("bad sleep want %v got %v", test.want, got)
+		}
+	}
+
+}
+
+func TestSetRetries(t *testing.T) {
+	p := New().SetRetries(18)
+	if p.retries != 18 {
+		t.Errorf("didn't set")
+	}
+}
+
+func TestSetPacer(t *testing.T) {
+	p := New().SetPacer(AmazonCloudDrivePacer)
+	if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.acdPacer) {
+		t.Errorf("calculatePace is not acdPacer")
+	}
+	p.SetPacer(GoogleDrivePacer)
+	if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.drivePacer) {
+		t.Errorf("calculatePace is not drivePacer")
+	}
+	p.SetPacer(DefaultPacer)
+	if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.defaultPacer) {
+		t.Errorf("calculatePace is not defaultPacer")
+	}
+}
+
+// emptyTokens empties the pacer of all its tokens
+func emptyTokens(p *Pacer) {
+	for len(p.pacer) != 0 {
+		<-p.pacer
+	}
+	for len(p.connTokens) != 0 {
+		<-p.connTokens
+	}
+}
+
+// waitForPace waits for duration for the pace to arrive
+// returns the time that it arrived or a zero time
+func waitForPace(p *Pacer, duration time.Duration) (when time.Time) {
+	select {
+	case <-time.After(duration):
+		return
+	case <-p.pacer:
+		return time.Now()
+	}
+}
+
+func TestBeginCall(t *testing.T) {
+	p := New().SetMaxConnections(10).SetMinSleep(1 * time.Millisecond)
+	emptyTokens(p)
+	go p.beginCall()
+	if !waitForPace(p, 10*time.Millisecond).IsZero() {
+		t.Errorf("beginSleep fired too early #1")
+	}
+	startTime := time.Now()
+	p.pacer <- struct{}{}
+	time.Sleep(1 * time.Millisecond)
+	connTime := time.Now()
+	p.connTokens <- struct{}{}
+	time.Sleep(1 * time.Millisecond)
+	paceTime := waitForPace(p, 1000*time.Millisecond)
+	if paceTime.IsZero() {
+		t.Errorf("beginSleep didn't fire")
+	} else if paceTime.Sub(startTime) < 0 {
+		t.Errorf("pace arrived before returning pace token")
+	} else if paceTime.Sub(connTime) < 0 {
+		t.Errorf("pace arrived before sending conn token")
+	}
+}
+
+func TestBeginCallZeroConnections(t *testing.T) {
+	p := New().SetMaxConnections(0).SetMinSleep(1 * time.Millisecond)
+	emptyTokens(p)
+	go p.beginCall()
+	if !waitForPace(p, 10*time.Millisecond).IsZero() {
+		t.Errorf("beginSleep fired too early #1")
+	}
+	startTime := time.Now()
+	p.pacer <- struct{}{}
+	time.Sleep(1 * time.Millisecond)
+	paceTime := waitForPace(p, 1000*time.Millisecond)
+	if paceTime.IsZero() {
+		t.Errorf("beginSleep didn't fire")
+	} else if paceTime.Sub(startTime) < 0 {
+		t.Errorf("pace arrived before returning pace token")
+	}
+}
+
+func TestDefaultPacer(t *testing.T) {
+	p := New().SetMinSleep(time.Millisecond).SetPacer(DefaultPacer).SetMaxSleep(time.Second).SetDecayConstant(2)
+	for _, test := range []struct {
+		in    time.Duration
+		retry bool
+		want  time.Duration
+	}{
+		{time.Millisecond, true, 2 * time.Millisecond},
+		{time.Second, true, time.Second},
+		{(3 * time.Second) / 4, true, time.Second},
+		{time.Second, false, 750 * time.Millisecond},
+		{1000 * time.Microsecond, false, time.Millisecond},
+		{1200 * time.Microsecond, false, time.Millisecond},
+	} {
+		p.sleepTime = test.in
+		p.defaultPacer(test.retry)
+		got := p.sleepTime
+		if got != test.want {
+			t.Errorf("bad sleep want %v got %v", test.want, got)
+		}
+	}
+
+}
+
+func TestAmazonCloudDrivePacer(t *testing.T) {
+	p := New().SetMinSleep(time.Millisecond).SetPacer(AmazonCloudDrivePacer).SetMaxSleep(time.Second).SetDecayConstant(2)
+	// Do lots of times because of the random number!
+	for _, test := range []struct {
+		in                 time.Duration
+		consecutiveRetries int
+		retry              bool
+		want               time.Duration
+	}{
+		{time.Millisecond, 0, true, time.Millisecond},
+		{10 * time.Millisecond, 0, true, time.Millisecond},
+		{1 * time.Second, 1, true, 500 * time.Millisecond},
+		{1 * time.Second, 2, true, 1 * time.Second},
+		{1 * time.Second, 3, true, 2 * time.Second},
+		{1 * time.Second, 4, true, 4 * time.Second},
+		{1 * time.Second, 5, true, 8 * time.Second},
+		{1 * time.Second, 6, true, 16 * time.Second},
+		{1 * time.Second, 7, true, 32 * time.Second},
+		{1 * time.Second, 8, true, 64 * time.Second},
+		{1 * time.Second, 9, true, 128 * time.Second},
+		{1 * time.Second, 10, true, 128 * time.Second},
+		{1 * time.Second, 11, true, 128 * time.Second},
+	} {
+		const n = 1000
+		var sum time.Duration
+		// measure average time over n cycles
+		for i := 0; i < n; i++ {
+			p.sleepTime = test.in
+			p.consecutiveRetries = test.consecutiveRetries
+			p.acdPacer(test.retry)
+			sum += p.sleepTime
+		}
+		got := sum / n
+		//t.Logf("%+v: got = %v", test, got)
+		if got < (test.want*9)/10 || got > (test.want*11)/10 {
+			t.Fatalf("%+v: bad sleep want %v+/-10%% got %v", test, test.want, got)
+		}
+	}
+}
+
+func TestGoogleDrivePacer(t *testing.T) {
+	p := New().SetMinSleep(time.Millisecond).SetPacer(GoogleDrivePacer).SetMaxSleep(time.Second).SetDecayConstant(2)
+	// Do lots of times because of the random number!
+	for _, test := range []struct {
+		in                 time.Duration
+		consecutiveRetries int
+		retry              bool
+		want               time.Duration
+	}{
+		{time.Millisecond, 0, true, time.Millisecond},
+		{10 * time.Millisecond, 0, true, time.Millisecond},
+		{1 * time.Second, 1, true, 1*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 2, true, 2*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 3, true, 4*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 4, true, 8*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 5, true, 16*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 6, true, 16*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 7, true, 16*time.Second + 500*time.Millisecond},
+	} {
+		const n = 1000
+		var sum time.Duration
+		// measure average time over n cycles
+		for i := 0; i < n; i++ {
+			p.sleepTime = test.in
+			p.consecutiveRetries = test.consecutiveRetries
+			p.drivePacer(test.retry)
+			sum += p.sleepTime
+		}
+		got := sum / n
+		//t.Logf("%+v: got = %v", test, got)
+		if got < (test.want*9)/10 || got > (test.want*11)/10 {
+			t.Fatalf("%+v: bad sleep want %v+/-10%% got %v", test, test.want, got)
+		}
+	}
+}
+
+func TestEndCall(t *testing.T) {
+	p := New().SetMaxConnections(5)
+	emptyTokens(p)
+	p.consecutiveRetries = 1
+	p.endCall(true)
+	if len(p.connTokens) != 1 {
+		t.Errorf("Expecting 1 token")
+	}
+	if p.consecutiveRetries != 2 {
+		t.Errorf("Bad consecutive retries")
+	}
+}
+
+func TestEndCallZeroConnections(t *testing.T) {
+	p := New().SetMaxConnections(0)
+	emptyTokens(p)
+	p.consecutiveRetries = 1
+	p.endCall(false)
+	if len(p.connTokens) != 0 {
+		t.Errorf("Expecting 0 token")
+	}
+	if p.consecutiveRetries != 0 {
+		t.Errorf("Bad consecutive retries")
+	}
+}
+
+var errFoo = errors.New("foo")
+
+type dummyPaced struct {
+	retry  bool
+	called int
+}
+
+func (dp *dummyPaced) fn() (bool, error) {
+	dp.called++
+	return dp.retry, errFoo
+}
+
+func Test_callNoRetry(t *testing.T) {
+	p := New().SetMinSleep(time.Millisecond).SetMaxSleep(2 * time.Millisecond)
+
+	dp := &dummyPaced{retry: false}
+	err := p.call(dp.fn, 10)
+	if dp.called != 1 {
+		t.Errorf("called want %d got %d", 1, dp.called)
+	}
+	if err != errFoo {
+		t.Errorf("err want %v got %v", errFoo, err)
+	}
+}
+
+func Test_callRetry(t *testing.T) {
+	p := New().SetMinSleep(time.Millisecond).SetMaxSleep(2 * time.Millisecond)
+
+	dp := &dummyPaced{retry: true}
+	err := p.call(dp.fn, 10)
+	if dp.called != 10 {
+		t.Errorf("called want %d got %d", 10, dp.called)
+	}
+	if err == errFoo {
+		t.Errorf("err didn't want %v got %v", errFoo, err)
+	}
+	_, ok := err.(fserrors.Retrier)
+	if !ok {
+		t.Errorf("didn't return a retry error")
+	}
+}
+
+func TestCall(t *testing.T) {
+	p := New().SetMinSleep(time.Millisecond).SetMaxSleep(2 * time.Millisecond).SetRetries(20)
+
+	dp := &dummyPaced{retry: true}
+	err := p.Call(dp.fn)
+	if dp.called != 20 {
+		t.Errorf("called want %d got %d", 20, dp.called)
+	}
+	_, ok := err.(fserrors.Retrier)
+	if !ok {
+		t.Errorf("didn't return a retry error")
+	}
+}
+
+func TestCallNoRetry(t *testing.T) {
+	p := New().SetMinSleep(time.Millisecond).SetMaxSleep(2 * time.Millisecond).SetRetries(20)
+
+	dp := &dummyPaced{retry: true}
+	err := p.CallNoRetry(dp.fn)
+	if dp.called != 1 {
+		t.Errorf("called want %d got %d", 1, dp.called)
+	}
+	_, ok := err.(fserrors.Retrier)
+	if !ok {
+		t.Errorf("didn't return a retry error")
+	}
+}

.rclone_repo/lib/pacer/tokens.go

@@ -0,0 +1,31 @@
+// Tokens for controlling concurrency
+
+package pacer
+
+// TokenDispenser is for controlling concurrency
+type TokenDispenser struct {
+	tokens chan struct{}
+}
+
+// NewTokenDispenser makes a pool of n tokens
+func NewTokenDispenser(n int) *TokenDispenser {
+	td := &TokenDispenser{
+		tokens: make(chan struct{}, n),
+	}
+	// Fill up the upload tokens
+	for i := 0; i < n; i++ {
+		td.tokens <- struct{}{}
+	}
+	return td
+}
+
+// Get gets a token from the pool - don't forget to return it with Put
+func (td *TokenDispenser) Get() {
+	<-td.tokens
+	return
+}
+
+// Put returns a token
+func (td *TokenDispenser) Put() {
+	td.tokens <- struct{}{}
+}

.rclone_repo/lib/pacer/tokens_test.go

@@ -0,0 +1,16 @@
+package pacer
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTokenDispenser(t *testing.T) {
+	td := NewTokenDispenser(5)
+	assert.Equal(t, 5, len(td.tokens))
+	td.Get()
+	assert.Equal(t, 4, len(td.tokens))
+	td.Put()
+	assert.Equal(t, 5, len(td.tokens))
+}