overdue
This commit is contained in:
bel
552
.rclone_repo/fs/walk/walk.go
Executable file
552
.rclone_repo/fs/walk/walk.go
Executable file
@@ -0,0 +1,552 @@
|
||||
// Package walk walks directories
|
||||
package walk
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/filter"
|
||||
"github.com/ncw/rclone/fs/list"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// ErrorSkipDir is used as a return value from Walk to indicate that the
|
||||
// directory named in the call is to be skipped. It is not returned as
|
||||
// an error by any function.
|
||||
var ErrorSkipDir = errors.New("skip this directory")
|
||||
|
||||
// ErrorCantListR is returned by WalkR if the underlying Fs isn't
|
||||
// capable of doing a recursive listing.
|
||||
var ErrorCantListR = errors.New("recursive directory listing not available")
|
||||
|
||||
// Func is the type of the function called for directory
|
||||
// visited by Walk. The path argument contains remote path to the directory.
|
||||
//
|
||||
// If there was a problem walking to directory named by path, the
|
||||
// incoming error will describe the problem and the function can
|
||||
// decide how to handle that error (and Walk will not descend into
|
||||
// that directory). If an error is returned, processing stops. The
|
||||
// sole exception is when the function returns the special value
|
||||
// ErrorSkipDir. If the function returns ErrorSkipDir, Walk skips the
|
||||
// directory's contents entirely.
|
||||
type Func func(path string, entries fs.DirEntries, err error) error
|
||||
|
||||
// Walk lists the directory.
|
||||
//
|
||||
// If includeAll is not set it will use the filters defined.
|
||||
//
|
||||
// If maxLevel is < 0 then it will recurse indefinitely, else it will
|
||||
// only do maxLevel levels.
|
||||
//
|
||||
// It calls fn for each tranche of DirEntries read.
|
||||
//
|
||||
// Note that fn will not be called concurrently whereas the directory
|
||||
// listing will proceed concurrently.
|
||||
//
|
||||
// Parent directories are always listed before their children
|
||||
//
|
||||
// This is implemented by WalkR if Config.UseRecursiveListing is true
|
||||
// and f supports it and level > 1, or WalkN otherwise.
|
||||
//
|
||||
// NB (f, path) to be replaced by fs.Dir at some point
|
||||
func Walk(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
|
||||
if (maxLevel < 0 || maxLevel > 1) && fs.Config.UseListR && f.Features().ListR != nil {
|
||||
return walkListR(f, path, includeAll, maxLevel, fn)
|
||||
}
|
||||
return walkListDirSorted(f, path, includeAll, maxLevel, fn)
|
||||
}
|
||||
|
||||
// walkListDirSorted lists the directory.
|
||||
//
|
||||
// It implements Walk using non recursive directory listing.
|
||||
func walkListDirSorted(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
|
||||
return walk(f, path, includeAll, maxLevel, fn, list.DirSorted)
|
||||
}
|
||||
|
||||
// walkListR lists the directory.
|
||||
//
|
||||
// It implements Walk using recursive directory listing if
|
||||
// available, or returns ErrorCantListR if not.
|
||||
func walkListR(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
|
||||
listR := f.Features().ListR
|
||||
if listR == nil {
|
||||
return ErrorCantListR
|
||||
}
|
||||
return walkR(f, path, includeAll, maxLevel, fn, listR)
|
||||
}
|
||||
|
||||
type listDirFunc func(fs fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error)
|
||||
|
||||
func walk(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listDir listDirFunc) error {
|
||||
var (
|
||||
wg sync.WaitGroup // sync closing of go routines
|
||||
traversing sync.WaitGroup // running directory traversals
|
||||
doClose sync.Once // close the channel once
|
||||
mu sync.Mutex // stop fn being called concurrently
|
||||
)
|
||||
// listJob describe a directory listing that needs to be done
|
||||
type listJob struct {
|
||||
remote string
|
||||
depth int
|
||||
}
|
||||
|
||||
in := make(chan listJob, fs.Config.Checkers)
|
||||
errs := make(chan error, 1)
|
||||
quit := make(chan struct{})
|
||||
closeQuit := func() {
|
||||
doClose.Do(func() {
|
||||
close(quit)
|
||||
go func() {
|
||||
for range in {
|
||||
traversing.Done()
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
for i := 0; i < fs.Config.Checkers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case job, ok := <-in:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
entries, err := listDir(f, includeAll, job.remote)
|
||||
var jobs []listJob
|
||||
if err == nil && job.depth != 0 {
|
||||
entries.ForDir(func(dir fs.Directory) {
|
||||
// Recurse for the directory
|
||||
jobs = append(jobs, listJob{
|
||||
remote: dir.Remote(),
|
||||
depth: job.depth - 1,
|
||||
})
|
||||
})
|
||||
}
|
||||
mu.Lock()
|
||||
err = fn(job.remote, entries, err)
|
||||
mu.Unlock()
|
||||
// NB once we have passed entries to fn we mustn't touch it again
|
||||
if err != nil && err != ErrorSkipDir {
|
||||
traversing.Done()
|
||||
fs.CountError(err)
|
||||
fs.Errorf(job.remote, "error listing: %v", err)
|
||||
closeQuit()
|
||||
// Send error to error channel if space
|
||||
select {
|
||||
case errs <- err:
|
||||
default:
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err == nil && len(jobs) > 0 {
|
||||
traversing.Add(len(jobs))
|
||||
go func() {
|
||||
// Now we have traversed this directory, send these
|
||||
// jobs off for traversal in the background
|
||||
for _, newJob := range jobs {
|
||||
in <- newJob
|
||||
}
|
||||
}()
|
||||
}
|
||||
traversing.Done()
|
||||
case <-quit:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
// Start the process
|
||||
traversing.Add(1)
|
||||
in <- listJob{
|
||||
remote: path,
|
||||
depth: maxLevel - 1,
|
||||
}
|
||||
traversing.Wait()
|
||||
close(in)
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
// return the first error returned or nil
|
||||
return <-errs
|
||||
}
|
||||
|
||||
// DirTree is a map of directories to entries
|
||||
type DirTree map[string]fs.DirEntries
|
||||
|
||||
// parentDir finds the parent directory of path
|
||||
func parentDir(entryPath string) string {
|
||||
dirPath := path.Dir(entryPath)
|
||||
if dirPath == "." {
|
||||
dirPath = ""
|
||||
}
|
||||
return dirPath
|
||||
}
|
||||
|
||||
// add an entry to the tree
|
||||
func (dt DirTree) add(entry fs.DirEntry) {
|
||||
dirPath := parentDir(entry.Remote())
|
||||
dt[dirPath] = append(dt[dirPath], entry)
|
||||
}
|
||||
|
||||
// add a directory entry to the tree
|
||||
func (dt DirTree) addDir(entry fs.DirEntry) {
|
||||
dt.add(entry)
|
||||
// create the directory itself if it doesn't exist already
|
||||
dirPath := entry.Remote()
|
||||
if _, ok := dt[dirPath]; !ok {
|
||||
dt[dirPath] = nil
|
||||
}
|
||||
}
|
||||
|
||||
// Find returns the DirEntry for filePath or nil if not found
|
||||
func (dt DirTree) Find(filePath string) (parentPath string, entry fs.DirEntry) {
|
||||
parentPath = parentDir(filePath)
|
||||
for _, entry := range dt[parentPath] {
|
||||
if entry.Remote() == filePath {
|
||||
return parentPath, entry
|
||||
}
|
||||
}
|
||||
return parentPath, nil
|
||||
}
|
||||
|
||||
// check that dirPath has a *Dir in its parent
|
||||
func (dt DirTree) checkParent(root, dirPath string) {
|
||||
if dirPath == root {
|
||||
return
|
||||
}
|
||||
parentPath, entry := dt.Find(dirPath)
|
||||
if entry != nil {
|
||||
return
|
||||
}
|
||||
dt[parentPath] = append(dt[parentPath], fs.NewDir(dirPath, time.Now()))
|
||||
dt.checkParent(root, parentPath)
|
||||
}
|
||||
|
||||
// check every directory in the tree has *Dir in its parent
|
||||
func (dt DirTree) checkParents(root string) {
|
||||
for dirPath := range dt {
|
||||
dt.checkParent(root, dirPath)
|
||||
}
|
||||
}
|
||||
|
||||
// Sort sorts all the Entries
|
||||
func (dt DirTree) Sort() {
|
||||
for _, entries := range dt {
|
||||
sort.Stable(entries)
|
||||
}
|
||||
}
|
||||
|
||||
// Dirs returns the directories in sorted order
|
||||
func (dt DirTree) Dirs() (dirNames []string) {
|
||||
for dirPath := range dt {
|
||||
dirNames = append(dirNames, dirPath)
|
||||
}
|
||||
sort.Strings(dirNames)
|
||||
return dirNames
|
||||
}
|
||||
|
||||
// Prune remove directories from a directory tree. dirNames contains
|
||||
// all directories to remove as keys, with true as values. dirNames
|
||||
// will be modified in the function.
|
||||
func (dt DirTree) Prune(dirNames map[string]bool) error {
|
||||
// We use map[string]bool to avoid recursion (and potential
|
||||
// stack exhaustion).
|
||||
|
||||
// First we need delete directories from their parents.
|
||||
for dName, remove := range dirNames {
|
||||
if !remove {
|
||||
// Currently all values should be
|
||||
// true, therefore this should not
|
||||
// happen. But this makes function
|
||||
// more predictable.
|
||||
fs.Infof(dName, "Directory in the map for prune, but the value is false")
|
||||
continue
|
||||
}
|
||||
if dName == "" {
|
||||
// if dName is root, do nothing (no parent exist)
|
||||
continue
|
||||
}
|
||||
parent := parentDir(dName)
|
||||
// It may happen that dt does not have a dName key,
|
||||
// since directory was excluded based on a filter. In
|
||||
// such case the loop will be skipped.
|
||||
for i, entry := range dt[parent] {
|
||||
switch x := entry.(type) {
|
||||
case fs.Directory:
|
||||
if x.Remote() == dName {
|
||||
// the slice is not sorted yet
|
||||
// to delete item
|
||||
// a) replace it with the last one
|
||||
dt[parent][i] = dt[parent][len(dt[parent])-1]
|
||||
// b) remove last
|
||||
dt[parent] = dt[parent][:len(dt[parent])-1]
|
||||
// we modify a slice within a loop, but we stop
|
||||
// iterating immediately
|
||||
break
|
||||
}
|
||||
case fs.Object:
|
||||
// do nothing
|
||||
default:
|
||||
return errors.Errorf("unknown object type %T", entry)
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for len(dirNames) > 0 {
|
||||
// According to golang specs, if new keys were added
|
||||
// during range iteration, they may be skipped.
|
||||
for dName, remove := range dirNames {
|
||||
if !remove {
|
||||
fs.Infof(dName, "Directory in the map for prune, but the value is false")
|
||||
continue
|
||||
}
|
||||
// First, add all subdirectories to dirNames.
|
||||
|
||||
// It may happen that dt[dName] does not exist.
|
||||
// If so, the loop will be skipped.
|
||||
for _, entry := range dt[dName] {
|
||||
switch x := entry.(type) {
|
||||
case fs.Directory:
|
||||
excludeDir := x.Remote()
|
||||
dirNames[excludeDir] = true
|
||||
case fs.Object:
|
||||
// do nothing
|
||||
default:
|
||||
return errors.Errorf("unknown object type %T", entry)
|
||||
|
||||
}
|
||||
}
|
||||
// Then remove current directory from DirTree
|
||||
delete(dt, dName)
|
||||
// and from dirNames
|
||||
delete(dirNames, dName)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// String emits a simple representation of the DirTree
|
||||
func (dt DirTree) String() string {
|
||||
out := new(bytes.Buffer)
|
||||
for _, dir := range dt.Dirs() {
|
||||
_, _ = fmt.Fprintf(out, "%s/\n", dir)
|
||||
for _, entry := range dt[dir] {
|
||||
flag := ""
|
||||
if _, ok := entry.(fs.Directory); ok {
|
||||
flag = "/"
|
||||
}
|
||||
_, _ = fmt.Fprintf(out, " %s%s\n", path.Base(entry.Remote()), flag)
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func walkRDirTree(f fs.Fs, startPath string, includeAll bool, maxLevel int, listR fs.ListRFn) (DirTree, error) {
|
||||
dirs := make(DirTree)
|
||||
// Entries can come in arbitrary order. We use toPrune to keep
|
||||
// all directories to exclude later.
|
||||
toPrune := make(map[string]bool)
|
||||
includeDirectory := filter.Active.IncludeDirectory(f)
|
||||
var mu sync.Mutex
|
||||
err := listR(startPath, func(entries fs.DirEntries) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for _, entry := range entries {
|
||||
slashes := strings.Count(entry.Remote(), "/")
|
||||
switch x := entry.(type) {
|
||||
case fs.Object:
|
||||
// Make sure we don't delete excluded files if not required
|
||||
if includeAll || filter.Active.IncludeObject(x) {
|
||||
if maxLevel < 0 || slashes <= maxLevel-1 {
|
||||
dirs.add(x)
|
||||
} else {
|
||||
// Make sure we include any parent directories of excluded objects
|
||||
dirPath := x.Remote()
|
||||
for ; slashes > maxLevel-1; slashes-- {
|
||||
dirPath = parentDir(dirPath)
|
||||
}
|
||||
dirs.checkParent(startPath, dirPath)
|
||||
}
|
||||
} else {
|
||||
fs.Debugf(x, "Excluded from sync (and deletion)")
|
||||
}
|
||||
// Check if we need to prune a directory later.
|
||||
if !includeAll && len(filter.Active.Opt.ExcludeFile) > 0 {
|
||||
basename := path.Base(x.Remote())
|
||||
if basename == filter.Active.Opt.ExcludeFile {
|
||||
excludeDir := parentDir(x.Remote())
|
||||
toPrune[excludeDir] = true
|
||||
fs.Debugf(basename, "Excluded from sync (and deletion) based on exclude file")
|
||||
}
|
||||
}
|
||||
case fs.Directory:
|
||||
inc, err := includeDirectory(x.Remote())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if includeAll || inc {
|
||||
if maxLevel < 0 || slashes <= maxLevel-1 {
|
||||
if slashes == maxLevel-1 {
|
||||
// Just add the object if at maxLevel
|
||||
dirs.add(x)
|
||||
} else {
|
||||
dirs.addDir(x)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fs.Debugf(x, "Excluded from sync (and deletion)")
|
||||
}
|
||||
default:
|
||||
return errors.Errorf("unknown object type %T", entry)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dirs.checkParents(startPath)
|
||||
if len(dirs) == 0 {
|
||||
dirs[startPath] = nil
|
||||
}
|
||||
err = dirs.Prune(toPrune)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dirs.Sort()
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
// Create a DirTree using List
|
||||
func walkNDirTree(f fs.Fs, path string, includeAll bool, maxLevel int, listDir listDirFunc) (DirTree, error) {
|
||||
dirs := make(DirTree)
|
||||
fn := func(dirPath string, entries fs.DirEntries, err error) error {
|
||||
if err == nil {
|
||||
dirs[dirPath] = entries
|
||||
}
|
||||
return err
|
||||
}
|
||||
err := walk(f, path, includeAll, maxLevel, fn, listDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
// NewDirTree returns a DirTree filled with the directory listing
|
||||
// using the parameters supplied.
|
||||
//
|
||||
// If includeAll is not set it will use the filters defined.
|
||||
//
|
||||
// If maxLevel is < 0 then it will recurse indefinitely, else it will
|
||||
// only do maxLevel levels.
|
||||
//
|
||||
// This is implemented by WalkR if Config.UseRecursiveListing is true
|
||||
// and f supports it and level > 1, or WalkN otherwise.
|
||||
//
|
||||
// NB (f, path) to be replaced by fs.Dir at some point
|
||||
func NewDirTree(f fs.Fs, path string, includeAll bool, maxLevel int) (DirTree, error) {
|
||||
if ListR := f.Features().ListR; (maxLevel < 0 || maxLevel > 1) && fs.Config.UseListR && ListR != nil {
|
||||
return walkRDirTree(f, path, includeAll, maxLevel, ListR)
|
||||
}
|
||||
return walkNDirTree(f, path, includeAll, maxLevel, list.DirSorted)
|
||||
}
|
||||
|
||||
func walkR(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listR fs.ListRFn) error {
|
||||
dirs, err := walkRDirTree(f, path, includeAll, maxLevel, listR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
skipping := false
|
||||
skipPrefix := ""
|
||||
emptyDir := fs.DirEntries{}
|
||||
for _, dirPath := range dirs.Dirs() {
|
||||
if skipping {
|
||||
// Skip over directories as required
|
||||
if strings.HasPrefix(dirPath, skipPrefix) {
|
||||
continue
|
||||
}
|
||||
skipping = false
|
||||
}
|
||||
entries := dirs[dirPath]
|
||||
if entries == nil {
|
||||
entries = emptyDir
|
||||
}
|
||||
err = fn(dirPath, entries, nil)
|
||||
if err == ErrorSkipDir {
|
||||
skipping = true
|
||||
skipPrefix = dirPath
|
||||
if skipPrefix != "" {
|
||||
skipPrefix += "/"
|
||||
}
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAll runs Walk getting all the results
|
||||
func GetAll(f fs.Fs, path string, includeAll bool, maxLevel int) (objs []fs.Object, dirs []fs.Directory, err error) {
|
||||
err = Walk(f, path, includeAll, maxLevel, func(dirPath string, entries fs.DirEntries, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
switch x := entry.(type) {
|
||||
case fs.Object:
|
||||
objs = append(objs, x)
|
||||
case fs.Directory:
|
||||
dirs = append(dirs, x)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// ListRHelper is used in the implementation of ListR to accumulate DirEntries
|
||||
type ListRHelper struct {
|
||||
callback fs.ListRCallback
|
||||
entries fs.DirEntries
|
||||
}
|
||||
|
||||
// NewListRHelper should be called from ListR with the callback passed in
|
||||
func NewListRHelper(callback fs.ListRCallback) *ListRHelper {
|
||||
return &ListRHelper{
|
||||
callback: callback,
|
||||
}
|
||||
}
|
||||
|
||||
// send sends the stored entries to the callback if there are >= max
|
||||
// entries.
|
||||
func (lh *ListRHelper) send(max int) (err error) {
|
||||
if len(lh.entries) >= max {
|
||||
err = lh.callback(lh.entries)
|
||||
lh.entries = lh.entries[:0]
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Add an entry to the stored entries and send them if there are more
|
||||
// than a certain amount
|
||||
func (lh *ListRHelper) Add(entry fs.DirEntry) error {
|
||||
if entry == nil {
|
||||
return nil
|
||||
}
|
||||
lh.entries = append(lh.entries, entry)
|
||||
return lh.send(100)
|
||||
}
|
||||
|
||||
// Flush the stored entries (if any) sending them to the callback
|
||||
func (lh *ListRHelper) Flush() error {
|
||||
return lh.send(1)
|
||||
}
|
||||
636
.rclone_repo/fs/walk/walk_test.go
Executable file
636
.rclone_repo/fs/walk/walk_test.go
Executable file
@@ -0,0 +1,636 @@
|
||||
package walk
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/filter"
|
||||
"github.com/ncw/rclone/fstest/mockdir"
|
||||
"github.com/ncw/rclone/fstest/mockobject"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type (
|
||||
listResult struct {
|
||||
entries fs.DirEntries
|
||||
err error
|
||||
}
|
||||
|
||||
listResults map[string]listResult
|
||||
|
||||
errorMap map[string]error
|
||||
|
||||
listDirs struct {
|
||||
mu sync.Mutex
|
||||
t *testing.T
|
||||
fs fs.Fs
|
||||
includeAll bool
|
||||
results listResults
|
||||
walkResults listResults
|
||||
walkErrors errorMap
|
||||
finalError error
|
||||
checkMaps bool
|
||||
maxLevel int
|
||||
}
|
||||
)
|
||||
|
||||
func newListDirs(t *testing.T, f fs.Fs, includeAll bool, results listResults, walkErrors errorMap, finalError error) *listDirs {
|
||||
return &listDirs{
|
||||
t: t,
|
||||
fs: f,
|
||||
includeAll: includeAll,
|
||||
results: results,
|
||||
walkErrors: walkErrors,
|
||||
walkResults: listResults{},
|
||||
finalError: finalError,
|
||||
checkMaps: true,
|
||||
maxLevel: -1,
|
||||
}
|
||||
}
|
||||
|
||||
// NoCheckMaps marks the maps as to be ignored at the end
|
||||
func (ls *listDirs) NoCheckMaps() *listDirs {
|
||||
ls.checkMaps = false
|
||||
return ls
|
||||
}
|
||||
|
||||
// SetLevel(1) turns off recursion
|
||||
func (ls *listDirs) SetLevel(maxLevel int) *listDirs {
|
||||
ls.maxLevel = maxLevel
|
||||
return ls
|
||||
}
|
||||
|
||||
// ListDir returns the expected listing for the directory
|
||||
func (ls *listDirs) ListDir(f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
|
||||
ls.mu.Lock()
|
||||
defer ls.mu.Unlock()
|
||||
assert.Equal(ls.t, ls.fs, f)
|
||||
assert.Equal(ls.t, ls.includeAll, includeAll)
|
||||
|
||||
// Fetch results for this path
|
||||
result, ok := ls.results[dir]
|
||||
if !ok {
|
||||
ls.t.Errorf("Unexpected list of %q", dir)
|
||||
return nil, errors.New("unexpected list")
|
||||
}
|
||||
delete(ls.results, dir)
|
||||
|
||||
// Put expected results for call of WalkFn
|
||||
ls.walkResults[dir] = result
|
||||
|
||||
return result.entries, result.err
|
||||
}
|
||||
|
||||
// ListR returns the expected listing for the directory using ListR
|
||||
func (ls *listDirs) ListR(dir string, callback fs.ListRCallback) (err error) {
|
||||
ls.mu.Lock()
|
||||
defer ls.mu.Unlock()
|
||||
|
||||
var errorReturn error
|
||||
for dirPath, result := range ls.results {
|
||||
// Put expected results for call of WalkFn
|
||||
// Note that we don't call the function at all if we got an error
|
||||
if result.err != nil {
|
||||
errorReturn = result.err
|
||||
}
|
||||
if errorReturn == nil {
|
||||
err = callback(result.entries)
|
||||
require.NoError(ls.t, err)
|
||||
ls.walkResults[dirPath] = result
|
||||
}
|
||||
}
|
||||
ls.results = listResults{}
|
||||
return errorReturn
|
||||
}
|
||||
|
||||
// IsFinished checks everything expected was used up
|
||||
func (ls *listDirs) IsFinished() {
|
||||
if ls.checkMaps {
|
||||
assert.Equal(ls.t, errorMap{}, ls.walkErrors)
|
||||
assert.Equal(ls.t, listResults{}, ls.results)
|
||||
assert.Equal(ls.t, listResults{}, ls.walkResults)
|
||||
}
|
||||
}
|
||||
|
||||
// WalkFn is called by the walk to test the expectations
|
||||
func (ls *listDirs) WalkFn(dir string, entries fs.DirEntries, err error) error {
|
||||
ls.mu.Lock()
|
||||
defer ls.mu.Unlock()
|
||||
// ls.t.Logf("WalkFn(%q, %v, %q)", dir, entries, err)
|
||||
|
||||
// Fetch expected entries and err
|
||||
result, ok := ls.walkResults[dir]
|
||||
if !ok {
|
||||
ls.t.Errorf("Unexpected walk of %q (result not found)", dir)
|
||||
return errors.New("result not found")
|
||||
}
|
||||
delete(ls.walkResults, dir)
|
||||
|
||||
// Check arguments are as expected
|
||||
assert.Equal(ls.t, result.entries, entries)
|
||||
assert.Equal(ls.t, result.err, err)
|
||||
|
||||
// Fetch return value
|
||||
returnErr, ok := ls.walkErrors[dir]
|
||||
if !ok {
|
||||
ls.t.Errorf("Unexpected walk of %q (error not found)", dir)
|
||||
return errors.New("error not found")
|
||||
}
|
||||
delete(ls.walkErrors, dir)
|
||||
|
||||
return returnErr
|
||||
}
|
||||
|
||||
// Walk does the walk and tests the expectations
|
||||
func (ls *listDirs) Walk() {
|
||||
err := walk(nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListDir)
|
||||
assert.Equal(ls.t, ls.finalError, err)
|
||||
ls.IsFinished()
|
||||
}
|
||||
|
||||
// WalkR does the walkR and tests the expectations
|
||||
func (ls *listDirs) WalkR() {
|
||||
err := walkR(nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListR)
|
||||
assert.Equal(ls.t, ls.finalError, err)
|
||||
if ls.finalError == nil {
|
||||
ls.IsFinished()
|
||||
}
|
||||
}
|
||||
|
||||
func testWalkEmpty(t *testing.T) *listDirs {
|
||||
return newListDirs(t, nil, false,
|
||||
listResults{
|
||||
"": {entries: fs.DirEntries{}, err: nil},
|
||||
},
|
||||
errorMap{
|
||||
"": nil,
|
||||
},
|
||||
nil,
|
||||
)
|
||||
}
|
||||
func TestWalkEmpty(t *testing.T) { testWalkEmpty(t).Walk() }
|
||||
func TestWalkREmpty(t *testing.T) { testWalkEmpty(t).WalkR() }
|
||||
|
||||
func testWalkEmptySkip(t *testing.T) *listDirs {
|
||||
return newListDirs(t, nil, true,
|
||||
listResults{
|
||||
"": {entries: fs.DirEntries{}, err: nil},
|
||||
},
|
||||
errorMap{
|
||||
"": ErrorSkipDir,
|
||||
},
|
||||
nil,
|
||||
)
|
||||
}
|
||||
func TestWalkEmptySkip(t *testing.T) { testWalkEmptySkip(t).Walk() }
|
||||
func TestWalkREmptySkip(t *testing.T) { testWalkEmptySkip(t).WalkR() }
|
||||
|
||||
func testWalkNotFound(t *testing.T) *listDirs {
|
||||
return newListDirs(t, nil, true,
|
||||
listResults{
|
||||
"": {err: fs.ErrorDirNotFound},
|
||||
},
|
||||
errorMap{
|
||||
"": fs.ErrorDirNotFound,
|
||||
},
|
||||
fs.ErrorDirNotFound,
|
||||
)
|
||||
}
|
||||
func TestWalkNotFound(t *testing.T) { testWalkNotFound(t).Walk() }
|
||||
func TestWalkRNotFound(t *testing.T) { testWalkNotFound(t).WalkR() }
|
||||
|
||||
func TestWalkNotFoundMaskError(t *testing.T) {
|
||||
// this doesn't work for WalkR
|
||||
newListDirs(t, nil, true,
|
||||
listResults{
|
||||
"": {err: fs.ErrorDirNotFound},
|
||||
},
|
||||
errorMap{
|
||||
"": nil,
|
||||
},
|
||||
nil,
|
||||
).Walk()
|
||||
}
|
||||
|
||||
func TestWalkNotFoundSkipkError(t *testing.T) {
|
||||
// this doesn't work for WalkR
|
||||
newListDirs(t, nil, true,
|
||||
listResults{
|
||||
"": {err: fs.ErrorDirNotFound},
|
||||
},
|
||||
errorMap{
|
||||
"": ErrorSkipDir,
|
||||
},
|
||||
nil,
|
||||
).Walk()
|
||||
}
|
||||
|
||||
func testWalkLevels(t *testing.T, maxLevel int) *listDirs {
|
||||
da := mockdir.New("a")
|
||||
oA := mockobject.Object("A")
|
||||
db := mockdir.New("a/b")
|
||||
oB := mockobject.Object("a/B")
|
||||
dc := mockdir.New("a/b/c")
|
||||
oC := mockobject.Object("a/b/C")
|
||||
dd := mockdir.New("a/b/c/d")
|
||||
oD := mockobject.Object("a/b/c/D")
|
||||
return newListDirs(t, nil, false,
|
||||
listResults{
|
||||
"": {entries: fs.DirEntries{oA, da}, err: nil},
|
||||
"a": {entries: fs.DirEntries{oB, db}, err: nil},
|
||||
"a/b": {entries: fs.DirEntries{oC, dc}, err: nil},
|
||||
"a/b/c": {entries: fs.DirEntries{oD, dd}, err: nil},
|
||||
"a/b/c/d": {entries: fs.DirEntries{}, err: nil},
|
||||
},
|
||||
errorMap{
|
||||
"": nil,
|
||||
"a": nil,
|
||||
"a/b": nil,
|
||||
"a/b/c": nil,
|
||||
"a/b/c/d": nil,
|
||||
},
|
||||
nil,
|
||||
).SetLevel(maxLevel)
|
||||
}
|
||||
func TestWalkLevels(t *testing.T) { testWalkLevels(t, -1).Walk() }
|
||||
func TestWalkRLevels(t *testing.T) { testWalkLevels(t, -1).WalkR() }
|
||||
func TestWalkLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).Walk() }
|
||||
func TestWalkRLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).WalkR() }
|
||||
|
||||
func TestWalkNDirTree(t *testing.T) {
|
||||
ls := testWalkLevels(t, -1)
|
||||
entries, err := walkNDirTree(nil, "", ls.includeAll, ls.maxLevel, ls.ListDir)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, `/
|
||||
A
|
||||
a/
|
||||
a/
|
||||
B
|
||||
b/
|
||||
a/b/
|
||||
C
|
||||
c/
|
||||
a/b/c/
|
||||
D
|
||||
d/
|
||||
a/b/c/d/
|
||||
`, entries.String())
|
||||
}
|
||||
|
||||
func testWalkLevelsNoRecursive(t *testing.T) *listDirs {
|
||||
da := mockdir.New("a")
|
||||
oA := mockobject.Object("A")
|
||||
return newListDirs(t, nil, false,
|
||||
listResults{
|
||||
"": {entries: fs.DirEntries{oA, da}, err: nil},
|
||||
},
|
||||
errorMap{
|
||||
"": nil,
|
||||
},
|
||||
nil,
|
||||
).SetLevel(1)
|
||||
}
|
||||
func TestWalkLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).Walk() }
|
||||
func TestWalkRLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).WalkR() }
|
||||
|
||||
func testWalkLevels2(t *testing.T) *listDirs {
|
||||
da := mockdir.New("a")
|
||||
oA := mockobject.Object("A")
|
||||
db := mockdir.New("a/b")
|
||||
oB := mockobject.Object("a/B")
|
||||
return newListDirs(t, nil, false,
|
||||
listResults{
|
||||
"": {entries: fs.DirEntries{oA, da}, err: nil},
|
||||
"a": {entries: fs.DirEntries{oB, db}, err: nil},
|
||||
},
|
||||
errorMap{
|
||||
"": nil,
|
||||
"a": nil,
|
||||
},
|
||||
nil,
|
||||
).SetLevel(2)
|
||||
}
|
||||
func TestWalkLevels2(t *testing.T) { testWalkLevels2(t).Walk() }
|
||||
func TestWalkRLevels2(t *testing.T) { testWalkLevels2(t).WalkR() }
|
||||
|
||||
func testWalkSkip(t *testing.T) *listDirs {
|
||||
da := mockdir.New("a")
|
||||
db := mockdir.New("a/b")
|
||||
dc := mockdir.New("a/b/c")
|
||||
return newListDirs(t, nil, false,
|
||||
listResults{
|
||||
"": {entries: fs.DirEntries{da}, err: nil},
|
||||
"a": {entries: fs.DirEntries{db}, err: nil},
|
||||
"a/b": {entries: fs.DirEntries{dc}, err: nil},
|
||||
},
|
||||
errorMap{
|
||||
"": nil,
|
||||
"a": nil,
|
||||
"a/b": ErrorSkipDir,
|
||||
},
|
||||
nil,
|
||||
)
|
||||
}
|
||||
func TestWalkSkip(t *testing.T) { testWalkSkip(t).Walk() }
|
||||
func TestWalkRSkip(t *testing.T) { testWalkSkip(t).WalkR() }
|
||||
|
||||
func testWalkErrors(t *testing.T) *listDirs {
|
||||
lr := listResults{}
|
||||
em := errorMap{}
|
||||
de := make(fs.DirEntries, 10)
|
||||
for i := range de {
|
||||
path := string('0' + i)
|
||||
de[i] = mockdir.New(path)
|
||||
lr[path] = listResult{entries: nil, err: fs.ErrorDirNotFound}
|
||||
em[path] = fs.ErrorDirNotFound
|
||||
}
|
||||
lr[""] = listResult{entries: de, err: nil}
|
||||
em[""] = nil
|
||||
return newListDirs(t, nil, true,
|
||||
lr,
|
||||
em,
|
||||
fs.ErrorDirNotFound,
|
||||
).NoCheckMaps()
|
||||
}
|
||||
func TestWalkErrors(t *testing.T) { testWalkErrors(t).Walk() }
|
||||
func TestWalkRErrors(t *testing.T) { testWalkErrors(t).WalkR() }
|
||||
|
||||
var errorBoom = errors.New("boom")
|
||||
|
||||
func makeTree(level int, terminalErrors bool) (listResults, errorMap) {
|
||||
lr := listResults{}
|
||||
em := errorMap{}
|
||||
var fill func(path string, level int)
|
||||
fill = func(path string, level int) {
|
||||
de := fs.DirEntries{}
|
||||
if level > 0 {
|
||||
for _, a := range "0123456789" {
|
||||
subPath := string(a)
|
||||
if path != "" {
|
||||
subPath = path + "/" + subPath
|
||||
}
|
||||
de = append(de, mockdir.New(subPath))
|
||||
fill(subPath, level-1)
|
||||
}
|
||||
}
|
||||
lr[path] = listResult{entries: de, err: nil}
|
||||
em[path] = nil
|
||||
if level == 0 && terminalErrors {
|
||||
em[path] = errorBoom
|
||||
}
|
||||
}
|
||||
fill("", level)
|
||||
return lr, em
|
||||
}
|
||||
|
||||
func testWalkMulti(t *testing.T) *listDirs {
|
||||
lr, em := makeTree(3, false)
|
||||
return newListDirs(t, nil, true,
|
||||
lr,
|
||||
em,
|
||||
nil,
|
||||
)
|
||||
}
|
||||
func TestWalkMulti(t *testing.T) { testWalkMulti(t).Walk() }
|
||||
func TestWalkRMulti(t *testing.T) { testWalkMulti(t).WalkR() }
|
||||
|
||||
func testWalkMultiErrors(t *testing.T) *listDirs {
|
||||
lr, em := makeTree(3, true)
|
||||
return newListDirs(t, nil, true,
|
||||
lr,
|
||||
em,
|
||||
errorBoom,
|
||||
).NoCheckMaps()
|
||||
}
|
||||
func TestWalkMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() }
|
||||
func TestWalkRMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() }
|
||||
|
||||
// a very simple listRcallback function
|
||||
func makeListRCallback(entries fs.DirEntries, err error) fs.ListRFn {
|
||||
return func(dir string, callback fs.ListRCallback) error {
|
||||
if err == nil {
|
||||
err = callback(entries)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkRDirTree(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
entries fs.DirEntries
|
||||
want string
|
||||
err error
|
||||
root string
|
||||
level int
|
||||
}{
|
||||
{fs.DirEntries{}, "/\n", nil, "", -1},
|
||||
{fs.DirEntries{mockobject.Object("a")}, `/
|
||||
a
|
||||
`, nil, "", -1},
|
||||
{fs.DirEntries{mockobject.Object("a/b")}, `/
|
||||
a/
|
||||
a/
|
||||
b
|
||||
`, nil, "", -1},
|
||||
{fs.DirEntries{mockobject.Object("a/b/c/d")}, `/
|
||||
a/
|
||||
a/
|
||||
b/
|
||||
a/b/
|
||||
c/
|
||||
a/b/c/
|
||||
d
|
||||
`, nil, "", -1},
|
||||
{fs.DirEntries{mockobject.Object("a")}, "", errorBoom, "", -1},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("0/1/2/3"),
|
||||
mockobject.Object("4/5/6/7"),
|
||||
mockobject.Object("8/9/a/b"),
|
||||
mockobject.Object("c/d/e/f"),
|
||||
mockobject.Object("g/h/i/j"),
|
||||
mockobject.Object("k/l/m/n"),
|
||||
mockobject.Object("o/p/q/r"),
|
||||
mockobject.Object("s/t/u/v"),
|
||||
mockobject.Object("w/x/y/z"),
|
||||
}, `/
|
||||
0/
|
||||
4/
|
||||
8/
|
||||
c/
|
||||
g/
|
||||
k/
|
||||
o/
|
||||
s/
|
||||
w/
|
||||
0/
|
||||
1/
|
||||
0/1/
|
||||
2/
|
||||
0/1/2/
|
||||
3
|
||||
4/
|
||||
5/
|
||||
4/5/
|
||||
6/
|
||||
4/5/6/
|
||||
7
|
||||
8/
|
||||
9/
|
||||
8/9/
|
||||
a/
|
||||
8/9/a/
|
||||
b
|
||||
c/
|
||||
d/
|
||||
c/d/
|
||||
e/
|
||||
c/d/e/
|
||||
f
|
||||
g/
|
||||
h/
|
||||
g/h/
|
||||
i/
|
||||
g/h/i/
|
||||
j
|
||||
k/
|
||||
l/
|
||||
k/l/
|
||||
m/
|
||||
k/l/m/
|
||||
n
|
||||
o/
|
||||
p/
|
||||
o/p/
|
||||
q/
|
||||
o/p/q/
|
||||
r
|
||||
s/
|
||||
t/
|
||||
s/t/
|
||||
u/
|
||||
s/t/u/
|
||||
v
|
||||
w/
|
||||
x/
|
||||
w/x/
|
||||
y/
|
||||
w/x/y/
|
||||
z
|
||||
`, nil, "", -1},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("a/b/c/d/e/f1"),
|
||||
mockobject.Object("a/b/c/d/e/f2"),
|
||||
mockobject.Object("a/b/c/d/e/f3"),
|
||||
}, `a/b/c/
|
||||
d/
|
||||
a/b/c/d/
|
||||
e/
|
||||
a/b/c/d/e/
|
||||
f1
|
||||
f2
|
||||
f3
|
||||
`, nil, "a/b/c", -1},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("A"),
|
||||
mockobject.Object("a/B"),
|
||||
mockobject.Object("a/b/C"),
|
||||
mockobject.Object("a/b/c/D"),
|
||||
mockobject.Object("a/b/c/d/E"),
|
||||
}, `/
|
||||
A
|
||||
a/
|
||||
a/
|
||||
B
|
||||
b/
|
||||
`, nil, "", 2},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("a/b/c"),
|
||||
mockobject.Object("a/b/c/d/e"),
|
||||
}, `/
|
||||
a/
|
||||
a/
|
||||
b/
|
||||
`, nil, "", 2},
|
||||
} {
|
||||
r, err := walkRDirTree(nil, test.root, true, test.level, makeListRCallback(test.entries, test.err))
|
||||
assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test))
|
||||
assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkRDirTreeExclude(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
entries fs.DirEntries
|
||||
want string
|
||||
err error
|
||||
root string
|
||||
level int
|
||||
excludeFile string
|
||||
includeAll bool
|
||||
}{
|
||||
{fs.DirEntries{mockobject.Object("a"), mockobject.Object("ignore")}, "", nil, "", -1, "ignore", false},
|
||||
{fs.DirEntries{mockobject.Object("a")}, `/
|
||||
a
|
||||
`, nil, "", -1, "ignore", false},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("a"),
|
||||
mockobject.Object("b/b"),
|
||||
mockobject.Object("b/.ignore"),
|
||||
}, `/
|
||||
a
|
||||
`, nil, "", -1, ".ignore", false},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("a"),
|
||||
mockobject.Object("b/.ignore"),
|
||||
mockobject.Object("b/b"),
|
||||
}, `/
|
||||
a
|
||||
b/
|
||||
b/
|
||||
.ignore
|
||||
b
|
||||
`, nil, "", -1, ".ignore", true},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("a"),
|
||||
mockobject.Object("b/b"),
|
||||
mockobject.Object("b/c/d/e"),
|
||||
mockobject.Object("b/c/ign"),
|
||||
mockobject.Object("b/c/x"),
|
||||
}, `/
|
||||
a
|
||||
b/
|
||||
b/
|
||||
b
|
||||
`, nil, "", -1, "ign", false},
|
||||
{fs.DirEntries{
|
||||
mockobject.Object("a"),
|
||||
mockobject.Object("b/b"),
|
||||
mockobject.Object("b/c/d/e"),
|
||||
mockobject.Object("b/c/ign"),
|
||||
mockobject.Object("b/c/x"),
|
||||
}, `/
|
||||
a
|
||||
b/
|
||||
b/
|
||||
b
|
||||
c/
|
||||
b/c/
|
||||
d/
|
||||
ign
|
||||
x
|
||||
b/c/d/
|
||||
e
|
||||
`, nil, "", -1, "ign", true},
|
||||
} {
|
||||
filter.Active.Opt.ExcludeFile = test.excludeFile
|
||||
r, err := walkRDirTree(nil, test.root, test.includeAll, test.level, makeListRCallback(test.entries, test.err))
|
||||
assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test))
|
||||
assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test))
|
||||
}
|
||||
// Set to default value, to avoid side effects
|
||||
filter.Active.Opt.ExcludeFile = ""
|
||||
}
|
||||
Reference in New Issue
Block a user