overdue

.rclone_repo/fs/hash/hash.go

@@ -0,0 +1,308 @@
+package hash
+
+import (
+	"crypto/md5"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"io"
+	"strings"
+
+	"github.com/ncw/rclone/backend/dropbox/dbhash"
+	"github.com/ncw/rclone/backend/onedrive/quickxorhash"
+	"github.com/pkg/errors"
+)
+
+// Type indicates a standard hashing algorithm
+type Type int
+
+// ErrUnsupported should be returned by filesystem,
+// if it is requested to deliver an unsupported hash type.
+var ErrUnsupported = errors.New("hash type not supported")
+
+const (
+	// MD5 indicates MD5 support
+	MD5 Type = 1 << iota
+
+	// SHA1 indicates SHA-1 support
+	SHA1
+
+	// Dropbox indicates Dropbox special hash
+	// https://www.dropbox.com/developers/reference/content-hash
+	Dropbox
+
+	// QuickXorHash indicates Microsoft onedrive hash
+	// https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
+	QuickXorHash
+
+	// None indicates no hashes are supported
+	None Type = 0
+)
+
+// Supported returns a set of all the supported hashes by
+// HashStream and MultiHasher.
+var Supported = NewHashSet(MD5, SHA1, Dropbox, QuickXorHash)
+
+// Width returns the width in characters for any HashType
+var Width = map[Type]int{
+	MD5:          32,
+	SHA1:         40,
+	Dropbox:      64,
+	QuickXorHash: 40,
+}
+
+// Stream will calculate hashes of all supported hash types.
+func Stream(r io.Reader) (map[Type]string, error) {
+	return StreamTypes(r, Supported)
+}
+
+// StreamTypes will calculate hashes of the requested hash types.
+func StreamTypes(r io.Reader, set Set) (map[Type]string, error) {
+	hashers, err := fromTypes(set)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = io.Copy(toMultiWriter(hashers), r)
+	if err != nil {
+		return nil, err
+	}
+	var ret = make(map[Type]string)
+	for k, v := range hashers {
+		ret[k] = hex.EncodeToString(v.Sum(nil))
+	}
+	return ret, nil
+}
+
+// String returns a string representation of the hash type.
+// The function will panic if the hash type is unknown.
+func (h Type) String() string {
+	switch h {
+	case None:
+		return "None"
+	case MD5:
+		return "MD5"
+	case SHA1:
+		return "SHA-1"
+	case Dropbox:
+		return "DropboxHash"
+	case QuickXorHash:
+		return "QuickXorHash"
+	default:
+		err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h))
+		panic(err)
+	}
+}
+
+// Set a Type from a flag
+func (h *Type) Set(s string) error {
+	switch s {
+	case "None":
+		*h = None
+	case "MD5":
+		*h = MD5
+	case "SHA-1":
+		*h = SHA1
+	case "DropboxHash":
+		*h = Dropbox
+	case "QuickXorHash":
+		*h = QuickXorHash
+	default:
+		return errors.Errorf("Unknown hash type %q", s)
+	}
+	return nil
+}
+
+// Type of the value
+func (h Type) Type() string {
+	return "string"
+}
+
+// fromTypes will return hashers for all the requested types.
+// The types must be a subset of SupportedHashes,
+// and this function must support all types.
+func fromTypes(set Set) (map[Type]hash.Hash, error) {
+	if !set.SubsetOf(Supported) {
+		return nil, errors.Errorf("requested set %08x contains unknown hash types", int(set))
+	}
+	var hashers = make(map[Type]hash.Hash)
+	types := set.Array()
+	for _, t := range types {
+		switch t {
+		case MD5:
+			hashers[t] = md5.New()
+		case SHA1:
+			hashers[t] = sha1.New()
+		case Dropbox:
+			hashers[t] = dbhash.New()
+		case QuickXorHash:
+			hashers[t] = quickxorhash.New()
+		default:
+			err := fmt.Sprintf("internal error: Unsupported hash type %v", t)
+			panic(err)
+		}
+	}
+	return hashers, nil
+}
+
+// toMultiWriter will return a set of hashers into a
+// single multiwriter, where one write will update all
+// the hashers.
+func toMultiWriter(h map[Type]hash.Hash) io.Writer {
+	// Convert to to slice
+	var w = make([]io.Writer, 0, len(h))
+	for _, v := range h {
+		w = append(w, v)
+	}
+	return io.MultiWriter(w...)
+}
+
+// A MultiHasher will construct various hashes on
+// all incoming writes.
+type MultiHasher struct {
+	w    io.Writer
+	size int64
+	h    map[Type]hash.Hash // Hashes
+}
+
+// NewMultiHasher will return a hash writer that will write all
+// supported hash types.
+func NewMultiHasher() *MultiHasher {
+	h, err := NewMultiHasherTypes(Supported)
+	if err != nil {
+		panic("internal error: could not create multihasher")
+	}
+	return h
+}
+
+// NewMultiHasherTypes will return a hash writer that will write
+// the requested hash types.
+func NewMultiHasherTypes(set Set) (*MultiHasher, error) {
+	hashers, err := fromTypes(set)
+	if err != nil {
+		return nil, err
+	}
+	m := MultiHasher{h: hashers, w: toMultiWriter(hashers)}
+	return &m, nil
+}
+
+func (m *MultiHasher) Write(p []byte) (n int, err error) {
+	n, err = m.w.Write(p)
+	m.size += int64(n)
+	return n, err
+}
+
+// Sums returns the sums of all accumulated hashes as hex encoded
+// strings.
+func (m *MultiHasher) Sums() map[Type]string {
+	dst := make(map[Type]string)
+	for k, v := range m.h {
+		dst[k] = hex.EncodeToString(v.Sum(nil))
+	}
+	return dst
+}
+
+// Size returns the number of bytes written
+func (m *MultiHasher) Size() int64 {
+	return m.size
+}
+
+// A Set Indicates one or more hash types.
+type Set int
+
+// NewHashSet will create a new hash set with the hash types supplied
+func NewHashSet(t ...Type) Set {
+	h := Set(None)
+	return h.Add(t...)
+}
+
+// Add one or more hash types to the set.
+// Returns the modified hash set.
+func (h *Set) Add(t ...Type) Set {
+	for _, v := range t {
+		*h |= Set(v)
+	}
+	return *h
+}
+
+// Contains returns true if the
+func (h Set) Contains(t Type) bool {
+	return int(h)&int(t) != 0
+}
+
+// Overlap returns the overlapping hash types
+func (h Set) Overlap(t Set) Set {
+	return Set(int(h) & int(t))
+}
+
+// SubsetOf will return true if all types of h
+// is present in the set c
+func (h Set) SubsetOf(c Set) bool {
+	return int(h)|int(c) == int(c)
+}
+
+// GetOne will return a hash type.
+// Currently the first is returned, but it could be
+// improved to return the strongest.
+func (h Set) GetOne() Type {
+	v := int(h)
+	i := uint(0)
+	for v != 0 {
+		if v&1 != 0 {
+			return Type(1 << i)
+		}
+		i++
+		v >>= 1
+	}
+	return Type(None)
+}
+
+// Array returns an array of all hash types in the set
+func (h Set) Array() (ht []Type) {
+	v := int(h)
+	i := uint(0)
+	for v != 0 {
+		if v&1 != 0 {
+			ht = append(ht, Type(1<<i))
+		}
+		i++
+		v >>= 1
+	}
+	return ht
+}
+
+// Count returns the number of hash types in the set
+func (h Set) Count() int {
+	if int(h) == 0 {
+		return 0
+	}
+	// credit: https://code.google.com/u/arnehormann/
+	x := uint64(h)
+	x -= (x >> 1) & 0x5555555555555555
+	x = (x>>2)&0x3333333333333333 + x&0x3333333333333333
+	x += x >> 4
+	x &= 0x0f0f0f0f0f0f0f0f
+	x *= 0x0101010101010101
+	return int(x >> 56)
+}
+
+// String returns a string representation of the hash set.
+// The function will panic if it contains an unknown type.
+func (h Set) String() string {
+	a := h.Array()
+	var r []string
+	for _, v := range a {
+		r = append(r, v.String())
+	}
+	return "[" + strings.Join(r, ", ") + "]"
+}
+
+// Equals checks to see if src == dst, but ignores empty strings
+// and returns true if either is empty.
+func Equals(src, dst string) bool {
+	if src == "" || dst == "" {
+		return true
+	}
+	return src == dst
+}

.rclone_repo/fs/hash/hash_test.go

@@ -0,0 +1,169 @@
+package hash_test
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/spf13/pflag"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Check it satisfies the interface
+var _ pflag.Value = (*hash.Type)(nil)
+
+func TestHashSet(t *testing.T) {
+	var h hash.Set
+
+	assert.Equal(t, 0, h.Count())
+
+	a := h.Array()
+	assert.Len(t, a, 0)
+
+	h = h.Add(hash.MD5)
+	assert.Equal(t, 1, h.Count())
+	assert.Equal(t, hash.MD5, h.GetOne())
+	a = h.Array()
+	assert.Len(t, a, 1)
+	assert.Equal(t, a[0], hash.MD5)
+
+	// Test overlap, with all hashes
+	h = h.Overlap(hash.Supported)
+	assert.Equal(t, 1, h.Count())
+	assert.Equal(t, hash.MD5, h.GetOne())
+	assert.True(t, h.SubsetOf(hash.Supported))
+	assert.True(t, h.SubsetOf(hash.NewHashSet(hash.MD5)))
+
+	h = h.Add(hash.SHA1)
+	assert.Equal(t, 2, h.Count())
+	one := h.GetOne()
+	if !(one == hash.MD5 || one == hash.SHA1) {
+		t.Fatalf("expected to be either MD5 or SHA1, got %v", one)
+	}
+	assert.True(t, h.SubsetOf(hash.Supported))
+	assert.False(t, h.SubsetOf(hash.NewHashSet(hash.MD5)))
+	assert.False(t, h.SubsetOf(hash.NewHashSet(hash.SHA1)))
+	assert.True(t, h.SubsetOf(hash.NewHashSet(hash.MD5, hash.SHA1)))
+	a = h.Array()
+	assert.Len(t, a, 2)
+
+	ol := h.Overlap(hash.NewHashSet(hash.MD5))
+	assert.Equal(t, 1, ol.Count())
+	assert.True(t, ol.Contains(hash.MD5))
+	assert.False(t, ol.Contains(hash.SHA1))
+
+	ol = h.Overlap(hash.NewHashSet(hash.MD5, hash.SHA1))
+	assert.Equal(t, 2, ol.Count())
+	assert.True(t, ol.Contains(hash.MD5))
+	assert.True(t, ol.Contains(hash.SHA1))
+}
+
+type hashTest struct {
+	input  []byte
+	output map[hash.Type]string
+}
+
+var hashTestSet = []hashTest{
+	{
+		input: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14},
+		output: map[hash.Type]string{
+			hash.MD5:          "bf13fc19e5151ac57d4252e0e0f87abe",
+			hash.SHA1:         "3ab6543c08a75f292a5ecedac87ec41642d12166",
+			hash.Dropbox:      "214d2fcf3566e94c99ad2f59bd993daca46d8521a0c447adf4b324f53fddc0c7",
+			hash.QuickXorHash: "0110c000085000031c0001095ec00218d0000700",
+		},
+	},
+	// Empty data set
+	{
+		input: []byte{},
+		output: map[hash.Type]string{
+			hash.MD5:          "d41d8cd98f00b204e9800998ecf8427e",
+			hash.SHA1:         "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+			hash.Dropbox:      "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+			hash.QuickXorHash: "0000000000000000000000000000000000000000",
+		},
+	},
+}
+
+func TestMultiHasher(t *testing.T) {
+	for _, test := range hashTestSet {
+		mh := hash.NewMultiHasher()
+		n, err := io.Copy(mh, bytes.NewBuffer(test.input))
+		require.NoError(t, err)
+		assert.Len(t, test.input, int(n))
+		sums := mh.Sums()
+		for k, v := range sums {
+			expect, ok := test.output[k]
+			require.True(t, ok, "test output for hash not found")
+			assert.Equal(t, expect, v)
+		}
+		// Test that all are present
+		for k, v := range test.output {
+			expect, ok := sums[k]
+			require.True(t, ok, "test output for hash not found")
+			assert.Equal(t, expect, v)
+		}
+	}
+}
+
+func TestMultiHasherTypes(t *testing.T) {
+	h := hash.SHA1
+	for _, test := range hashTestSet {
+		mh, err := hash.NewMultiHasherTypes(hash.NewHashSet(h))
+		if err != nil {
+			t.Fatal(err)
+		}
+		n, err := io.Copy(mh, bytes.NewBuffer(test.input))
+		require.NoError(t, err)
+		assert.Len(t, test.input, int(n))
+		sums := mh.Sums()
+		assert.Len(t, sums, 1)
+		assert.Equal(t, sums[h], test.output[h])
+	}
+}
+
+func TestHashStream(t *testing.T) {
+	for _, test := range hashTestSet {
+		sums, err := hash.Stream(bytes.NewBuffer(test.input))
+		require.NoError(t, err)
+		for k, v := range sums {
+			expect, ok := test.output[k]
+			require.True(t, ok)
+			assert.Equal(t, v, expect)
+		}
+		// Test that all are present
+		for k, v := range test.output {
+			expect, ok := sums[k]
+			require.True(t, ok)
+			assert.Equal(t, v, expect)
+		}
+	}
+}
+
+func TestHashStreamTypes(t *testing.T) {
+	h := hash.SHA1
+	for _, test := range hashTestSet {
+		sums, err := hash.StreamTypes(bytes.NewBuffer(test.input), hash.NewHashSet(h))
+		require.NoError(t, err)
+		assert.Len(t, sums, 1)
+		assert.Equal(t, sums[h], test.output[h])
+	}
+}
+
+func TestHashSetStringer(t *testing.T) {
+	h := hash.NewHashSet(hash.SHA1, hash.MD5, hash.Dropbox, hash.QuickXorHash)
+	assert.Equal(t, h.String(), "[MD5, SHA-1, DropboxHash, QuickXorHash]")
+	h = hash.NewHashSet(hash.SHA1)
+	assert.Equal(t, h.String(), "[SHA-1]")
+	h = hash.NewHashSet()
+	assert.Equal(t, h.String(), "[]")
+}
+
+func TestHashStringer(t *testing.T) {
+	h := hash.MD5
+	assert.Equal(t, h.String(), "MD5")
+	h = hash.None
+	assert.Equal(t, h.String(), "None")
+}