overdue

.rclone_repo/fs/list/list.go

@@ -0,0 +1,102 @@
+// Package list contains list functions
+package list
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/filter"
+	"github.com/pkg/errors"
+)
+
+// DirSorted reads Object and *Dir into entries for the given Fs.
+//
+// dir is the start directory, "" for root
+//
+// If includeAll is specified all files will be added, otherwise only
+// files and directories passing the filter will be added.
+//
+// Files will be returned in sorted order
+func DirSorted(f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
+	// Get unfiltered entries from the fs
+	entries, err = f.List(dir)
+	if err != nil {
+		return nil, err
+	}
+	// This should happen only if exclude files lives in the
+	// starting directory, otherwise ListDirSorted should not be
+	// called.
+	if !includeAll && filter.Active.ListContainsExcludeFile(entries) {
+		fs.Debugf(dir, "Excluded from sync (and deletion)")
+		return nil, nil
+	}
+	return filterAndSortDir(entries, includeAll, dir, filter.Active.IncludeObject, filter.Active.IncludeDirectory(f))
+}
+
+// filter (if required) and check the entries, then sort them
+func filterAndSortDir(entries fs.DirEntries, includeAll bool, dir string,
+	IncludeObject func(o fs.Object) bool,
+	IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) {
+	newEntries = entries[:0] // in place filter
+	prefix := ""
+	if dir != "" {
+		prefix = dir + "/"
+	}
+	for _, entry := range entries {
+		ok := true
+		// check includes and types
+		switch x := entry.(type) {
+		case fs.Object:
+			// Make sure we don't delete excluded files if not required
+			if !includeAll && !IncludeObject(x) {
+				ok = false
+				fs.Debugf(x, "Excluded from sync (and deletion)")
+			}
+		case fs.Directory:
+			if !includeAll {
+				include, err := IncludeDirectory(x.Remote())
+				if err != nil {
+					return nil, err
+				}
+				if !include {
+					ok = false
+					fs.Debugf(x, "Excluded from sync (and deletion)")
+				}
+			}
+		default:
+			return nil, errors.Errorf("unknown object type %T", entry)
+		}
+		// check remote name belongs in this directry
+		remote := entry.Remote()
+		switch {
+		case !ok:
+			// ignore
+		case !strings.HasPrefix(remote, prefix):
+			ok = false
+			fs.Errorf(entry, "Entry doesn't belong in directory %q (too short) - ignoring", dir)
+		case remote == prefix:
+			ok = false
+			fs.Errorf(entry, "Entry doesn't belong in directory %q (same as directory) - ignoring", dir)
+		case strings.ContainsRune(remote[len(prefix):], '/'):
+			ok = false
+			fs.Errorf(entry, "Entry doesn't belong in directory %q (contains subdir) - ignoring", dir)
+		default:
+			// ok
+		}
+		if ok {
+			newEntries = append(newEntries, entry)
+		}
+	}
+	entries = newEntries
+
+	// Sort the directory entries by Remote
+	//
+	// We use a stable sort here just in case there are
+	// duplicates. Assuming the remote delivers the entries in a
+	// consistent order, this will give the best user experience
+	// in syncing as it will use the first entry for the sync
+	// comparison.
+	sort.Stable(entries)
+	return entries, nil
+}

.rclone_repo/fs/list/list_test.go

@@ -0,0 +1,104 @@
+package list
+
+import (
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/mockdir"
+	"github.com/ncw/rclone/fstest/mockobject"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// NB integration tests for DirSorted are in
+// fs/operations/listdirsorted_test.go
+
+func TestFilterAndSortIncludeAll(t *testing.T) {
+	da := mockdir.New("a")
+	oA := mockobject.Object("A")
+	db := mockdir.New("b")
+	oB := mockobject.Object("B")
+	dc := mockdir.New("c")
+	oC := mockobject.Object("C")
+	dd := mockdir.New("d")
+	oD := mockobject.Object("D")
+	entries := fs.DirEntries{da, oA, db, oB, dc, oC, dd, oD}
+	includeObject := func(o fs.Object) bool {
+		return o != oB
+	}
+	includeDirectory := func(remote string) (bool, error) {
+		return remote != "c", nil
+	}
+	// no filter
+	newEntries, err := filterAndSortDir(entries, true, "", includeObject, includeDirectory)
+	require.NoError(t, err)
+	assert.Equal(t,
+		newEntries,
+		fs.DirEntries{oA, oB, oC, oD, da, db, dc, dd},
+	)
+	// filter
+	newEntries, err = filterAndSortDir(entries, false, "", includeObject, includeDirectory)
+	require.NoError(t, err)
+	assert.Equal(t,
+		newEntries,
+		fs.DirEntries{oA, oC, oD, da, db, dd},
+	)
+}
+
+func TestFilterAndSortCheckDir(t *testing.T) {
+	// Check the different kinds of error when listing "dir"
+	da := mockdir.New("dir/")
+	oA := mockobject.Object("diR/a")
+	db := mockdir.New("dir/b")
+	oB := mockobject.Object("dir/B/sub")
+	dc := mockdir.New("dir/c")
+	oC := mockobject.Object("dir/C")
+	dd := mockdir.New("dir/d")
+	oD := mockobject.Object("dir/D")
+	entries := fs.DirEntries{da, oA, db, oB, dc, oC, dd, oD}
+	newEntries, err := filterAndSortDir(entries, true, "dir", nil, nil)
+	require.NoError(t, err)
+	assert.Equal(t,
+		newEntries,
+		fs.DirEntries{oC, oD, db, dc, dd},
+	)
+}
+
+func TestFilterAndSortCheckDirRoot(t *testing.T) {
+	// Check the different kinds of error when listing the root ""
+	da := mockdir.New("")
+	oA := mockobject.Object("A")
+	db := mockdir.New("b")
+	oB := mockobject.Object("B/sub")
+	dc := mockdir.New("c")
+	oC := mockobject.Object("C")
+	dd := mockdir.New("d")
+	oD := mockobject.Object("D")
+	entries := fs.DirEntries{da, oA, db, oB, dc, oC, dd, oD}
+	newEntries, err := filterAndSortDir(entries, true, "", nil, nil)
+	require.NoError(t, err)
+	assert.Equal(t,
+		newEntries,
+		fs.DirEntries{oA, oC, oD, db, dc, dd},
+	)
+}
+
+type unknownDirEntry string
+
+func (o unknownDirEntry) String() string         { return string(o) }
+func (o unknownDirEntry) Remote() string         { return string(o) }
+func (o unknownDirEntry) ModTime() (t time.Time) { return t }
+func (o unknownDirEntry) Size() int64            { return 0 }
+
+func TestFilterAndSortUnknown(t *testing.T) {
+	// Check that an unknown entry produces an error
+	da := mockdir.New("")
+	oA := mockobject.Object("A")
+	ub := unknownDirEntry("b")
+	oB := mockobject.Object("B/sub")
+	entries := fs.DirEntries{da, oA, ub, oB}
+	newEntries, err := filterAndSortDir(entries, true, "", nil, nil)
+	assert.Error(t, err, "error")
+	assert.Nil(t, newEntries)
+}