overdue

.rclone_repo/cmd/lsf/lsf.go

@@ -0,0 +1,205 @@
+package lsf
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/cmd/ls/lshelp"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/fs/operations"
+	"github.com/ncw/rclone/fs/walk"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+var (
+	format    string
+	separator string
+	dirSlash  bool
+	recurse   bool
+	hashType  = hash.MD5
+	filesOnly bool
+	dirsOnly  bool
+	csv       bool
+	absolute  bool
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefintion)
+	flags := commandDefintion.Flags()
+	flags.StringVarP(&format, "format", "F", "p", "Output format - see  help for details")
+	flags.StringVarP(&separator, "separator", "s", ";", "Separator for the items in the format.")
+	flags.BoolVarP(&dirSlash, "dir-slash", "d", true, "Append a slash to directory names.")
+	flags.VarP(&hashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash")
+	flags.BoolVarP(&filesOnly, "files-only", "", false, "Only list files.")
+	flags.BoolVarP(&dirsOnly, "dirs-only", "", false, "Only list directories.")
+	flags.BoolVarP(&csv, "csv", "", false, "Output in CSV format.")
+	flags.BoolVarP(&absolute, "absolute", "", false, "Put a leading / in front of path names.")
+	commandDefintion.Flags().BoolVarP(&recurse, "recursive", "R", false, "Recurse into the listing.")
+}
+
+var commandDefintion = &cobra.Command{
+	Use:   "lsf remote:path",
+	Short: `List directories and objects in remote:path formatted for parsing`,
+	Long: `
+List the contents of the source path (directories and objects) to
+standard output in a form which is easy to parse by scripts.  By
+default this will just be the names of the objects and directories,
+one per line.  The directories will have a / suffix.
+
+Eg
+
+    $ rclone lsf swift:bucket
+    bevajer5jef
+    canole
+    diwogej7
+    ferejej3gux/
+    fubuwic
+
+Use the --format option to control what gets listed.  By default this
+is just the path, but you can use these parameters to control the
+output:
+
+    p - path
+    s - size
+    t - modification time
+    h - hash
+    i - ID of object if known
+    m - MimeType of object if known
+
+So if you wanted the path, size and modification time, you would use
+--format "pst", or maybe --format "tsp" to put the path last.
+
+Eg
+
+    $ rclone lsf  --format "tsp" swift:bucket
+    2016-06-25 18:55:41;60295;bevajer5jef
+    2016-06-25 18:55:43;90613;canole
+    2016-06-25 18:55:43;94467;diwogej7
+    2018-04-26 08:50:45;0;ferejej3gux/
+    2016-06-25 18:55:40;37600;fubuwic
+
+If you specify "h" in the format you will get the MD5 hash by default,
+use the "--hash" flag to change which hash you want.  Note that this
+can be returned as an empty string if it isn't available on the object
+(and for directories), "ERROR" if there was an error reading it from
+the object and "UNSUPPORTED" if that object does not support that hash
+type.
+
+For example to emulate the md5sum command you can use
+
+    rclone lsf -R --hash MD5 --format hp --separator "  " --files-only .
+
+Eg
+
+    $ rclone lsf -R --hash MD5 --format hp --separator "  " --files-only swift:bucket 
+    7908e352297f0f530b84a756f188baa3  bevajer5jef
+    cd65ac234e6fea5925974a51cdd865cc  canole
+    03b5341b4f234b9d984d03ad076bae91  diwogej7
+    8fd37c3810dd660778137ac3a66cc06d  fubuwic
+    99713e14a4c4ff553acaf1930fad985b  gixacuh7ku
+
+(Though "rclone md5sum ." is an easier way of typing this.)
+
+By default the separator is ";" this can be changed with the
+--separator flag.  Note that separators aren't escaped in the path so
+putting it last is a good strategy.
+
+Eg
+
+    $ rclone lsf  --separator "," --format "tshp" swift:bucket
+    2016-06-25 18:55:41,60295,7908e352297f0f530b84a756f188baa3,bevajer5jef
+    2016-06-25 18:55:43,90613,cd65ac234e6fea5925974a51cdd865cc,canole
+    2016-06-25 18:55:43,94467,03b5341b4f234b9d984d03ad076bae91,diwogej7
+    2018-04-26 08:52:53,0,,ferejej3gux/
+    2016-06-25 18:55:40,37600,8fd37c3810dd660778137ac3a66cc06d,fubuwic
+
+You can output in CSV standard format.  This will escape things in "
+if they contain ,
+
+Eg
+
+    $ rclone lsf --csv --files-only --format ps remote:path
+    test.log,22355
+    test.sh,449
+    "this file contains a comma, in the file name.txt",6
+
+Note that the --absolute parameter is useful for making lists of files
+to pass to an rclone copy with the --files-from flag.
+
+For example to find all the files modified within one day and copy
+those only (without traversing the whole directory structure):
+
+    rclone lsf --absolute --files-only --max-age 1d /path/to/local > new_files
+    rclone copy --files-from new_files /path/to/local remote:path
+
+` + lshelp.Help,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1, command, args)
+		fsrc := cmd.NewFsSrc(args)
+		cmd.Run(false, false, command, func() error {
+			// Work out if the separatorFlag was supplied or not
+			separatorFlag := command.Flags().Lookup("separator")
+			separatorFlagSupplied := separatorFlag != nil && separatorFlag.Changed
+			// Default the separator to , if using CSV
+			if csv && !separatorFlagSupplied {
+				separator = ","
+			}
+			return Lsf(fsrc, os.Stdout)
+		})
+	},
+}
+
+// Lsf lists all the objects in the path with modification time, size
+// and path in specific format.
+func Lsf(fsrc fs.Fs, out io.Writer) error {
+	var list operations.ListFormat
+	list.SetSeparator(separator)
+	list.SetCSV(csv)
+	list.SetDirSlash(dirSlash)
+	list.SetAbsolute(absolute)
+
+	for _, char := range format {
+		switch char {
+		case 'p':
+			list.AddPath()
+		case 't':
+			list.AddModTime()
+		case 's':
+			list.AddSize()
+		case 'h':
+			list.AddHash(hashType)
+		case 'i':
+			list.AddID()
+		case 'm':
+			list.AddMimeType()
+		default:
+			return errors.Errorf("Unknown format character %q", char)
+		}
+	}
+
+	return walk.Walk(fsrc, "", false, operations.ConfigMaxDepth(recurse), func(path string, entries fs.DirEntries, err error) error {
+		if err != nil {
+			fs.CountError(err)
+			fs.Errorf(path, "error listing: %v", err)
+			return nil
+		}
+		for _, entry := range entries {
+			_, isDir := entry.(fs.Directory)
+			if isDir {
+				if filesOnly {
+					continue
+				}
+			} else {
+				if dirsOnly {
+					continue
+				}
+			}
+			_, _ = fmt.Fprintln(out, list.Format(entry))
+		}
+		return nil
+	})
+}

.rclone_repo/cmd/lsf/lsf_test.go

@@ -0,0 +1,225 @@
+package lsf
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/list"
+	"github.com/ncw/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	_ "github.com/ncw/rclone/backend/local"
+)
+
+func TestDefaultLsf(t *testing.T) {
+	fstest.Initialise()
+	buf := new(bytes.Buffer)
+
+	f, err := fs.NewFs("testfiles")
+	require.NoError(t, err)
+
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `file1
+file2
+file3
+subdir/
+`, buf.String())
+}
+
+func TestRecurseFlag(t *testing.T) {
+	fstest.Initialise()
+	buf := new(bytes.Buffer)
+
+	f, err := fs.NewFs("testfiles")
+	require.NoError(t, err)
+
+	recurse = true
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `file1
+file2
+file3
+subdir/
+subdir/file1
+subdir/file2
+subdir/file3
+`, buf.String())
+	recurse = false
+}
+
+func TestDirSlashFlag(t *testing.T) {
+	fstest.Initialise()
+	buf := new(bytes.Buffer)
+
+	f, err := fs.NewFs("testfiles")
+	require.NoError(t, err)
+
+	dirSlash = true
+	format = "p"
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `file1
+file2
+file3
+subdir/
+`, buf.String())
+
+	buf = new(bytes.Buffer)
+	dirSlash = false
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `file1
+file2
+file3
+subdir
+`, buf.String())
+}
+
+func TestFormat(t *testing.T) {
+	fstest.Initialise()
+	f, err := fs.NewFs("testfiles")
+	require.NoError(t, err)
+
+	buf := new(bytes.Buffer)
+	format = "p"
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `file1
+file2
+file3
+subdir
+`, buf.String())
+
+	buf = new(bytes.Buffer)
+	format = "s"
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `0
+321
+1234
+-1
+`, buf.String())
+
+	buf = new(bytes.Buffer)
+	format = "hp"
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `d41d8cd98f00b204e9800998ecf8427e;file1
+409d6c19451dd39d4a94e42d2ff2c834;file2
+9b4c8a5e36d3be7e2c4b1d75ded8c8a1;file3
+;subdir
+`, buf.String())
+
+	buf = new(bytes.Buffer)
+	format = "p"
+	filesOnly = true
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `file1
+file2
+file3
+`, buf.String())
+	filesOnly = false
+
+	buf = new(bytes.Buffer)
+	format = "p"
+	dirsOnly = true
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `subdir
+`, buf.String())
+	dirsOnly = false
+
+	buf = new(bytes.Buffer)
+	format = "t"
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+
+	items, _ := list.DirSorted(f, true, "")
+	var expectedOutput string
+	for _, item := range items {
+		expectedOutput += item.ModTime().Format("2006-01-02 15:04:05") + "\n"
+	}
+
+	assert.Equal(t, expectedOutput, buf.String())
+
+	buf = new(bytes.Buffer)
+	format = "sp"
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `0;file1
+321;file2
+1234;file3
+-1;subdir
+`, buf.String())
+	format = ""
+}
+
+func TestSeparator(t *testing.T) {
+	fstest.Initialise()
+	f, err := fs.NewFs("testfiles")
+	require.NoError(t, err)
+	format = "ps"
+
+	buf := new(bytes.Buffer)
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `file1;0
+file2;321
+file3;1234
+subdir;-1
+`, buf.String())
+
+	separator = "__SEP__"
+	buf = new(bytes.Buffer)
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+	assert.Equal(t, `file1__SEP__0
+file2__SEP__321
+file3__SEP__1234
+subdir__SEP__-1
+`, buf.String())
+	format = ""
+	separator = ""
+}
+
+func TestWholeLsf(t *testing.T) {
+	fstest.Initialise()
+	f, err := fs.NewFs("testfiles")
+	require.NoError(t, err)
+	format = "pst"
+	separator = "_+_"
+	recurse = true
+	dirSlash = true
+
+	buf := new(bytes.Buffer)
+	err = Lsf(f, buf)
+	require.NoError(t, err)
+
+	items, _ := list.DirSorted(f, true, "")
+	itemsInSubdir, _ := list.DirSorted(f, true, "subdir")
+	var expectedOutput []string
+	for _, item := range items {
+		expectedOutput = append(expectedOutput, item.ModTime().Format("2006-01-02 15:04:05"))
+	}
+	for _, item := range itemsInSubdir {
+		expectedOutput = append(expectedOutput, item.ModTime().Format("2006-01-02 15:04:05"))
+	}
+
+	assert.Equal(t, `file1_+_0_+_`+expectedOutput[0]+`
+file2_+_321_+_`+expectedOutput[1]+`
+file3_+_1234_+_`+expectedOutput[2]+`
+subdir/_+_-1_+_`+expectedOutput[3]+`
+subdir/file1_+_0_+_`+expectedOutput[4]+`
+subdir/file2_+_1_+_`+expectedOutput[5]+`
+subdir/file3_+_111_+_`+expectedOutput[6]+`
+`, buf.String())
+
+	format = ""
+	separator = ""
+	recurse = false
+	dirSlash = false
+}