overdue

.rclone_repo/vfs/cache_test.go

@@ -0,0 +1,462 @@
+package vfs
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/djherbis/times"
+	"github.com/ncw/rclone/fstest"
+	"github.com/spf13/pflag"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Check CacheMode it satisfies the pflag interface
+var _ pflag.Value = (*CacheMode)(nil)
+
+func TestCacheModeString(t *testing.T) {
+	assert.Equal(t, "off", CacheModeOff.String())
+	assert.Equal(t, "full", CacheModeFull.String())
+	assert.Equal(t, "CacheMode(17)", CacheMode(17).String())
+}
+
+func TestCacheModeSet(t *testing.T) {
+	var m CacheMode
+
+	err := m.Set("full")
+	assert.NoError(t, err)
+	assert.Equal(t, CacheModeFull, m)
+
+	err = m.Set("potato")
+	assert.Error(t, err, "Unknown cache mode level")
+
+	err = m.Set("")
+	assert.Error(t, err, "Unknown cache mode level")
+}
+
+func TestCacheModeType(t *testing.T) {
+	var m CacheMode
+	assert.Equal(t, "string", m.Type())
+}
+
+// convert c.item to a string
+func itemAsString(c *cache) []string {
+	c.itemMu.Lock()
+	defer c.itemMu.Unlock()
+	var out []string
+	for name, item := range c.item {
+		out = append(out, fmt.Sprintf("name=%q isFile=%v opens=%d", name, item.isFile, item.opens))
+	}
+	sort.Strings(out)
+	return out
+}
+
+func TestCacheNew(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Disable the cache cleaner as it interferes with these tests
+	opt := DefaultOpt
+	opt.CachePollInterval = 0
+	c, err := newCache(ctx, r.Fremote, &opt)
+	require.NoError(t, err)
+
+	assert.Contains(t, c.root, "vfs")
+	assert.Contains(t, c.f.Root(), filepath.Base(r.Fremote.Root()))
+	assert.Equal(t, []string(nil), itemAsString(c))
+
+	// mkdir
+	p, err := c.mkdir("potato")
+	require.NoError(t, err)
+	assert.Equal(t, "potato", filepath.Base(p))
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0`,
+	}, itemAsString(c))
+
+	fi, err := os.Stat(filepath.Dir(p))
+	require.NoError(t, err)
+	assert.True(t, fi.IsDir())
+
+	// get
+	item := c.get("potato")
+	item2 := c.get("potato")
+	assert.Equal(t, item, item2)
+	assert.WithinDuration(t, time.Now(), item.atime, time.Second)
+
+	// updateTime
+	//.. before
+	t1 := time.Now().Add(-60 * time.Minute)
+	c.updateTime("potato", t1)
+	item = c.get("potato")
+	assert.NotEqual(t, t1, item.atime)
+	assert.Equal(t, 0, item.opens)
+	//..after
+	t2 := time.Now().Add(60 * time.Minute)
+	c.updateTime("potato", t2)
+	item = c.get("potato")
+	assert.Equal(t, t2, item.atime)
+	assert.Equal(t, 0, item.opens)
+
+	// open
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0`,
+		`name="potato" isFile=true opens=0`,
+	}, itemAsString(c))
+	c.open("/potato")
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=1`,
+		`name="potato" isFile=true opens=1`,
+	}, itemAsString(c))
+	item = c.get("potato")
+	assert.WithinDuration(t, time.Now(), item.atime, time.Second)
+	assert.Equal(t, 1, item.opens)
+
+	// write the file
+	err = ioutil.WriteFile(p, []byte("hello"), 0600)
+	require.NoError(t, err)
+
+	// read its atime
+	fi, err = os.Stat(p)
+	assert.NoError(t, err)
+	atime := times.Get(fi).AccessTime()
+
+	// updateAtimes
+	item = c.get("potato")
+	item.atime = time.Now().Add(-24 * time.Hour)
+	err = c.updateAtimes()
+	require.NoError(t, err)
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=1`,
+		`name="potato" isFile=true opens=1`,
+	}, itemAsString(c))
+	item = c.get("potato")
+	assert.Equal(t, atime, item.atime)
+
+	// updateAtimes - not in the cache
+	oldItem := item
+	c.itemMu.Lock()
+	delete(c.item, "potato") // remove from cache
+	c.itemMu.Unlock()
+	err = c.updateAtimes()
+	require.NoError(t, err)
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=1`,
+		`name="potato" isFile=true opens=0`,
+	}, itemAsString(c))
+	item = c.get("potato")
+	assert.Equal(t, atime, item.atime)
+	c.itemMu.Lock()
+	c.item["potato"] = oldItem // restore to cache
+	c.itemMu.Unlock()
+
+	// try purging with file open
+	c.purgeOld(10 * time.Second)
+	_, err = os.Stat(p)
+	assert.NoError(t, err)
+
+	// close
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=1`,
+		`name="potato" isFile=true opens=1`,
+	}, itemAsString(c))
+	c.updateTime("potato", t2)
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=1`,
+		`name="potato" isFile=true opens=1`,
+	}, itemAsString(c))
+	c.close("potato/")
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0`,
+		`name="potato" isFile=true opens=0`,
+	}, itemAsString(c))
+	item = c.get("potato")
+	assert.WithinDuration(t, time.Now(), item.atime, time.Second)
+	assert.Equal(t, 0, item.opens)
+
+	// try purging with file closed
+	c.purgeOld(10 * time.Second)
+	// ...nothing should happend
+	_, err = os.Stat(p)
+	assert.NoError(t, err)
+
+	//.. purge again with -ve age
+	c.purgeOld(-10 * time.Second)
+	_, err = os.Stat(p)
+	assert.True(t, os.IsNotExist(err))
+
+	// clean - have tested the internals already
+	c.clean()
+
+	// cleanup
+	err = c.cleanUp()
+	require.NoError(t, err)
+	_, err = os.Stat(c.root)
+	assert.True(t, os.IsNotExist(err))
+}
+
+func TestCacheOpens(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	c, err := newCache(ctx, r.Fremote, &DefaultOpt)
+	require.NoError(t, err)
+
+	assert.Equal(t, []string(nil), itemAsString(c))
+	c.open("potato")
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=1`,
+		`name="potato" isFile=true opens=1`,
+	}, itemAsString(c))
+	c.open("potato")
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=2`,
+		`name="potato" isFile=true opens=2`,
+	}, itemAsString(c))
+	c.close("potato")
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=1`,
+		`name="potato" isFile=true opens=1`,
+	}, itemAsString(c))
+	c.close("potato")
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0`,
+		`name="potato" isFile=true opens=0`,
+	}, itemAsString(c))
+
+	c.open("potato")
+	c.open("a//b/c/d/one")
+	c.open("a/b/c/d/e/two")
+	c.open("a/b/c/d/e/f/three")
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=4`,
+		`name="a" isFile=false opens=3`,
+		`name="a/b" isFile=false opens=3`,
+		`name="a/b/c" isFile=false opens=3`,
+		`name="a/b/c/d" isFile=false opens=3`,
+		`name="a/b/c/d/e" isFile=false opens=2`,
+		`name="a/b/c/d/e/f" isFile=false opens=1`,
+		`name="a/b/c/d/e/f/three" isFile=true opens=1`,
+		`name="a/b/c/d/e/two" isFile=true opens=1`,
+		`name="a/b/c/d/one" isFile=true opens=1`,
+		`name="potato" isFile=true opens=1`,
+	}, itemAsString(c))
+	c.close("potato")
+	c.close("a/b/c/d/one")
+	c.close("a/b/c/d/e/two")
+	c.close("a/b/c//d/e/f/three")
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0`,
+		`name="a" isFile=false opens=0`,
+		`name="a/b" isFile=false opens=0`,
+		`name="a/b/c" isFile=false opens=0`,
+		`name="a/b/c/d" isFile=false opens=0`,
+		`name="a/b/c/d/e" isFile=false opens=0`,
+		`name="a/b/c/d/e/f" isFile=false opens=0`,
+		`name="a/b/c/d/e/f/three" isFile=true opens=0`,
+		`name="a/b/c/d/e/two" isFile=true opens=0`,
+		`name="a/b/c/d/one" isFile=true opens=0`,
+		`name="potato" isFile=true opens=0`,
+	}, itemAsString(c))
+}
+
+// test the open, mkdir, purge, close, purge sequence
+func TestCacheOpenMkdir(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Disable the cache cleaner as it interferes with these tests
+	opt := DefaultOpt
+	opt.CachePollInterval = 0
+	c, err := newCache(ctx, r.Fremote, &opt)
+	require.NoError(t, err)
+
+	// open
+	c.open("sub/potato")
+
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=1`,
+		`name="sub" isFile=false opens=1`,
+		`name="sub/potato" isFile=true opens=1`,
+	}, itemAsString(c))
+
+	// mkdir
+	p, err := c.mkdir("sub/potato")
+	require.NoError(t, err)
+	assert.Equal(t, "potato", filepath.Base(p))
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=1`,
+		`name="sub" isFile=false opens=1`,
+		`name="sub/potato" isFile=true opens=1`,
+	}, itemAsString(c))
+
+	// test directory exists
+	fi, err := os.Stat(filepath.Dir(p))
+	require.NoError(t, err)
+	assert.True(t, fi.IsDir())
+
+	// clean the cache
+	c.purgeOld(-10 * time.Second)
+
+	// test directory still exists
+	fi, err = os.Stat(filepath.Dir(p))
+	require.NoError(t, err)
+	assert.True(t, fi.IsDir())
+
+	// close
+	c.close("sub/potato")
+
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0`,
+		`name="sub" isFile=false opens=0`,
+		`name="sub/potato" isFile=true opens=0`,
+	}, itemAsString(c))
+
+	// clean the cache
+	c.purgeOld(-10 * time.Second)
+
+	assert.Equal(t, []string(nil), itemAsString(c))
+
+	// test directory does not exist
+	fi, err = os.Stat(filepath.Dir(p))
+	require.True(t, os.IsNotExist(err))
+}
+
+func TestCacheCacheDir(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	c, err := newCache(ctx, r.Fremote, &DefaultOpt)
+	require.NoError(t, err)
+
+	assert.Equal(t, []string(nil), itemAsString(c))
+
+	c.cacheDir("dir")
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0`,
+		`name="dir" isFile=false opens=0`,
+	}, itemAsString(c))
+
+	c.cacheDir("dir/sub")
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0`,
+		`name="dir" isFile=false opens=0`,
+		`name="dir/sub" isFile=false opens=0`,
+	}, itemAsString(c))
+
+	c.cacheDir("dir/sub2/subsub2")
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0`,
+		`name="dir" isFile=false opens=0`,
+		`name="dir/sub" isFile=false opens=0`,
+		`name="dir/sub2" isFile=false opens=0`,
+		`name="dir/sub2/subsub2" isFile=false opens=0`,
+	}, itemAsString(c))
+}
+
+func TestCachePurgeOld(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	c, err := newCache(ctx, r.Fremote, &DefaultOpt)
+	require.NoError(t, err)
+
+	// Test funcs
+	var removed []string
+	removedDir := true
+	removeFile := func(name string) {
+		removed = append(removed, name)
+	}
+	removeDir := func(name string) bool {
+		if removedDir {
+			removed = append(removed, name+"/")
+		}
+		return removedDir
+	}
+
+	removed = nil
+	c._purgeOld(-10*time.Second, removeFile, removeDir)
+	assert.Equal(t, []string(nil), removed)
+
+	c.open("sub/dir2/potato2")
+	c.open("sub/dir/potato")
+	c.close("sub/dir2/potato2")
+	c.open("sub/dir/potato")
+
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=2`,
+		`name="sub" isFile=false opens=2`,
+		`name="sub/dir" isFile=false opens=2`,
+		`name="sub/dir/potato" isFile=true opens=2`,
+		`name="sub/dir2" isFile=false opens=0`,
+		`name="sub/dir2/potato2" isFile=true opens=0`,
+	}, itemAsString(c))
+
+	removed = nil
+	removedDir = true
+	c._purgeOld(-10*time.Second, removeFile, removeDir)
+	assert.Equal(t, []string{
+		"sub/dir2/potato2",
+		"sub/dir2/",
+	}, removed)
+
+	c.close("sub/dir/potato")
+
+	removed = nil
+	removedDir = true
+	c._purgeOld(-10*time.Second, removeFile, removeDir)
+	assert.Equal(t, []string(nil), removed)
+
+	c.close("sub/dir/potato")
+
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0`,
+		`name="sub" isFile=false opens=0`,
+		`name="sub/dir" isFile=false opens=0`,
+		`name="sub/dir/potato" isFile=true opens=0`,
+	}, itemAsString(c))
+
+	removed = nil
+	removedDir = false
+	c._purgeOld(10*time.Second, removeFile, removeDir)
+	assert.Equal(t, []string(nil), removed)
+
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0`,
+		`name="sub" isFile=false opens=0`,
+		`name="sub/dir" isFile=false opens=0`,
+		`name="sub/dir/potato" isFile=true opens=0`,
+	}, itemAsString(c))
+
+	removed = nil
+	removedDir = true
+	c._purgeOld(-10*time.Second, removeFile, removeDir)
+	assert.Equal(t, []string{
+		"sub/dir/potato",
+		"sub/dir/",
+		"sub/",
+		"/",
+	}, removed)
+
+	assert.Equal(t, []string(nil), itemAsString(c))
+}