overdue

.rclone_repo/cmd/info/all.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+exec rclone --check-normalization=true --check-control=true --check-length=true info \
+	/tmp/testInfo \
+	TestAmazonCloudDrive:testInfo \
+	TestB2:testInfo \
+	TestCryptDrive:testInfo \
+	TestCryptSwift:testInfo \
+	TestDrive:testInfo \
+	TestDropbox:testInfo \
+	TestGoogleCloudStorage:rclone-testinfo \
+	TestOneDrive:testInfo \
+	TestS3:rclone-testinfo \
+	TestSftp:testInfo \
+	TestSwift:testInfo \
+	TestYandex:testInfo \
+	TestFTP:testInfo
+
+#	TestHubic:testInfo \

.rclone_repo/cmd/info/info.go

@@ -0,0 +1,269 @@
+package info
+
+// FIXME once translations are implemented will need a no-escape
+// option for Put so we can make these tests work agaig
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/fs/object"
+	"github.com/ncw/rclone/fstest"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+var (
+	checkNormalization bool
+	checkControl       bool
+	checkLength        bool
+	checkStreaming     bool
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefintion)
+	commandDefintion.Flags().BoolVarP(&checkNormalization, "check-normalization", "", true, "Check UTF-8 Normalization.")
+	commandDefintion.Flags().BoolVarP(&checkControl, "check-control", "", true, "Check control characters.")
+	commandDefintion.Flags().BoolVarP(&checkLength, "check-length", "", true, "Check max filename length.")
+	commandDefintion.Flags().BoolVarP(&checkStreaming, "check-streaming", "", true, "Check uploads with indeterminate file size.")
+}
+
+var commandDefintion = &cobra.Command{
+	Use:   "info [remote:path]+",
+	Short: `Discovers file name or other limitations for paths.`,
+	Long: `rclone info discovers what filenames and upload methods are possible
+to write to the paths passed in and how long they can be.  It can take some
+time.  It will write test files into the remote:path passed in.  It outputs
+a bit of go code for each one.
+`,
+	Hidden: true,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1E6, command, args)
+		for i := range args {
+			f := cmd.NewFsDir(args[i : i+1])
+			cmd.Run(false, false, command, func() error {
+				return readInfo(f)
+			})
+		}
+	},
+}
+
+type results struct {
+	f                    fs.Fs
+	mu                   sync.Mutex
+	charNeedsEscaping    map[rune]bool
+	maxFileLength        int
+	canWriteUnnormalized bool
+	canReadUnnormalized  bool
+	canReadRenormalized  bool
+	canStream            bool
+}
+
+func newResults(f fs.Fs) *results {
+	return &results{
+		f:                 f,
+		charNeedsEscaping: make(map[rune]bool),
+	}
+}
+
+// Print the results to stdout
+func (r *results) Print() {
+	fmt.Printf("// %s\n", r.f.Name())
+	if checkControl {
+		escape := []string{}
+		for c, needsEscape := range r.charNeedsEscaping {
+			if needsEscape {
+				escape = append(escape, fmt.Sprintf("0x%02X", c))
+			}
+		}
+		sort.Strings(escape)
+		fmt.Printf("charNeedsEscaping = []byte{\n")
+		fmt.Printf("\t%s\n", strings.Join(escape, ", "))
+		fmt.Printf("}\n")
+	}
+	if checkLength {
+		fmt.Printf("maxFileLength = %d\n", r.maxFileLength)
+	}
+	if checkNormalization {
+		fmt.Printf("canWriteUnnormalized = %v\n", r.canWriteUnnormalized)
+		fmt.Printf("canReadUnnormalized   = %v\n", r.canReadUnnormalized)
+		fmt.Printf("canReadRenormalized   = %v\n", r.canReadRenormalized)
+	}
+	if checkStreaming {
+		fmt.Printf("canStream = %v\n", r.canStream)
+	}
+}
+
+// writeFile writes a file with some random contents
+func (r *results) writeFile(path string) (fs.Object, error) {
+	contents := fstest.RandomString(50)
+	src := object.NewStaticObjectInfo(path, time.Now(), int64(len(contents)), true, nil, r.f)
+	return r.f.Put(bytes.NewBufferString(contents), src)
+}
+
+// check whether normalization is enforced and check whether it is
+// done on the files anyway
+func (r *results) checkUTF8Normalization() {
+	unnormalized := "Héroique"
+	normalized := "Héroique"
+	_, err := r.writeFile(unnormalized)
+	if err != nil {
+		r.canWriteUnnormalized = false
+		return
+	}
+	r.canWriteUnnormalized = true
+	_, err = r.f.NewObject(unnormalized)
+	if err == nil {
+		r.canReadUnnormalized = true
+	}
+	_, err = r.f.NewObject(normalized)
+	if err == nil {
+		r.canReadRenormalized = true
+	}
+}
+
+// check we can write file with the rune passed in
+func (r *results) checkChar(c rune) {
+	fs.Infof(r.f, "Writing file 0x%02X", c)
+	path := fmt.Sprintf("0x%02X-%c-", c, c)
+	_, err := r.writeFile(path)
+	escape := false
+	if err != nil {
+		fs.Infof(r.f, "Couldn't write file 0x%02X", c)
+	} else {
+		fs.Infof(r.f, "OK writing file 0x%02X", c)
+	}
+	r.mu.Lock()
+	r.charNeedsEscaping[c] = escape
+	r.mu.Unlock()
+}
+
+// check we can write a file with the control chars
+func (r *results) checkControls() {
+	fs.Infof(r.f, "Trying to create control character file names")
+	// Concurrency control
+	tokens := make(chan struct{}, fs.Config.Checkers)
+	for i := 0; i < fs.Config.Checkers; i++ {
+		tokens <- struct{}{}
+	}
+	var wg sync.WaitGroup
+	for i := rune(0); i < 128; i++ {
+		if i == 0 || i == '/' {
+			// We're not even going to check NULL or /
+			r.charNeedsEscaping[i] = true
+			continue
+		}
+		wg.Add(1)
+		c := i
+		go func() {
+			defer wg.Done()
+			token := <-tokens
+			r.checkChar(c)
+			tokens <- token
+		}()
+	}
+	wg.Wait()
+	fs.Infof(r.f, "Done trying to create control character file names")
+}
+
+// find the max file name size we can use
+func (r *results) findMaxLength() {
+	const maxLen = 16 * 1024
+	name := make([]byte, maxLen)
+	for i := range name {
+		name[i] = 'a'
+	}
+	// Find the first size of filename we can't write
+	i := sort.Search(len(name), func(i int) (fail bool) {
+		defer func() {
+			if err := recover(); err != nil {
+				fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
+				fail = true
+			}
+		}()
+
+		path := string(name[:i])
+		_, err := r.writeFile(path)
+		if err != nil {
+			fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
+			return true
+		}
+		fs.Infof(r.f, "Wrote file with name length %d", i)
+		return false
+	})
+	r.maxFileLength = i - 1
+	fs.Infof(r.f, "Max file length is %d", r.maxFileLength)
+}
+
+func (r *results) checkStreaming() {
+	putter := r.f.Put
+	if r.f.Features().PutStream != nil {
+		fs.Infof(r.f, "Given remote has specialized streaming function. Using that to test streaming.")
+		putter = r.f.Features().PutStream
+	}
+
+	contents := "thinking of test strings is hard"
+	buf := bytes.NewBufferString(contents)
+	hashIn := hash.NewMultiHasher()
+	in := io.TeeReader(buf, hashIn)
+
+	objIn := object.NewStaticObjectInfo("checkStreamingTest", time.Now(), -1, true, nil, r.f)
+	objR, err := putter(in, objIn)
+	if err != nil {
+		fs.Infof(r.f, "Streamed file failed to upload (%v)", err)
+		r.canStream = false
+		return
+	}
+
+	hashes := hashIn.Sums()
+	types := objR.Fs().Hashes().Array()
+	for _, Hash := range types {
+		sum, err := objR.Hash(Hash)
+		if err != nil {
+			fs.Infof(r.f, "Streamed file failed when getting hash %v (%v)", Hash, err)
+			r.canStream = false
+			return
+		}
+		if !hash.Equals(hashes[Hash], sum) {
+			fs.Infof(r.f, "Streamed file has incorrect hash %v: expecting %q got %q", Hash, hashes[Hash], sum)
+			r.canStream = false
+			return
+		}
+	}
+	if int64(len(contents)) != objR.Size() {
+		fs.Infof(r.f, "Streamed file has incorrect file size: expecting %d got %d", len(contents), objR.Size())
+		r.canStream = false
+		return
+	}
+	r.canStream = true
+}
+
+func readInfo(f fs.Fs) error {
+	err := f.Mkdir("")
+	if err != nil {
+		return errors.Wrap(err, "couldn't mkdir")
+	}
+	r := newResults(f)
+	if checkControl {
+		r.checkControls()
+	}
+	if checkLength {
+		r.findMaxLength()
+	}
+	if checkNormalization {
+		r.checkUTF8Normalization()
+	}
+	if checkStreaming {
+		r.checkStreaming()
+	}
+	r.Print()
+	return nil
+}