This commit is contained in:
bel
2020-01-13 03:37:51 +00:00
commit c8eb52f9ba
2023 changed files with 702080 additions and 0 deletions

195
.rclone_repo/cmd/mount/dir.go Executable file
View File

@@ -0,0 +1,195 @@
// +build linux darwin freebsd
package mount
import (
"os"
"time"
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs/log"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
)
// Dir represents a directory entry
type Dir struct {
*vfs.Dir
}
// Check interface satsified
var _ fusefs.Node = (*Dir)(nil)
// Attr updates the attributes of a directory
func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) (err error) {
defer log.Trace(d, "")("attr=%+v, err=%v", a, &err)
a.Valid = mountlib.AttrTimeout
a.Gid = d.VFS().Opt.GID
a.Uid = d.VFS().Opt.UID
a.Mode = os.ModeDir | d.VFS().Opt.DirPerms
modTime := d.ModTime()
a.Atime = modTime
a.Mtime = modTime
a.Ctime = modTime
a.Crtime = modTime
// FIXME include Valid so get some caching?
// FIXME fs.Debugf(d.path, "Dir.Attr %+v", a)
return nil
}
// Check interface satisfied
var _ fusefs.NodeSetattrer = (*Dir)(nil)
// Setattr handles attribute changes from FUSE. Currently supports ModTime only.
func (d *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) (err error) {
defer log.Trace(d, "stat=%+v", req)("err=%v", &err)
if d.VFS().Opt.NoModTime {
return nil
}
if req.Valid.MtimeNow() {
err = d.SetModTime(time.Now())
} else if req.Valid.Mtime() {
err = d.SetModTime(req.Mtime)
}
return translateError(err)
}
// Check interface satisfied
var _ fusefs.NodeRequestLookuper = (*Dir)(nil)
// Lookup looks up a specific entry in the receiver.
//
// Lookup should return a Node corresponding to the entry. If the
// name does not exist in the directory, Lookup should return ENOENT.
//
// Lookup need not to handle the names "." and "..".
func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fusefs.Node, err error) {
defer log.Trace(d, "name=%q", req.Name)("node=%+v, err=%v", &node, &err)
mnode, err := d.Dir.Stat(req.Name)
if err != nil {
return nil, translateError(err)
}
resp.EntryValid = mountlib.AttrTimeout
switch x := mnode.(type) {
case *vfs.File:
return &File{x}, nil
case *vfs.Dir:
return &Dir{x}, nil
}
panic("bad type")
}
// Check interface satisfied
var _ fusefs.HandleReadDirAller = (*Dir)(nil)
// ReadDirAll reads the contents of the directory
func (d *Dir) ReadDirAll(ctx context.Context) (dirents []fuse.Dirent, err error) {
itemsRead := -1
defer log.Trace(d, "")("item=%d, err=%v", &itemsRead, &err)
items, err := d.Dir.ReadDirAll()
if err != nil {
return nil, translateError(err)
}
for _, node := range items {
var dirent = fuse.Dirent{
// Inode FIXME ???
Type: fuse.DT_File,
Name: node.Name(),
}
if node.IsDir() {
dirent.Type = fuse.DT_Dir
}
dirents = append(dirents, dirent)
}
itemsRead = len(dirents)
return dirents, nil
}
var _ fusefs.NodeCreater = (*Dir)(nil)
// Create makes a new file
func (d *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (node fusefs.Node, handle fusefs.Handle, err error) {
defer log.Trace(d, "name=%q", req.Name)("node=%v, handle=%v, err=%v", &node, &handle, &err)
file, err := d.Dir.Create(req.Name, int(req.Flags))
if err != nil {
return nil, nil, translateError(err)
}
fh, err := file.Open(int(req.Flags) | os.O_CREATE)
if err != nil {
return nil, nil, translateError(err)
}
return &File{file}, &FileHandle{fh}, err
}
var _ fusefs.NodeMkdirer = (*Dir)(nil)
// Mkdir creates a new directory
func (d *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (node fusefs.Node, err error) {
defer log.Trace(d, "name=%q", req.Name)("node=%+v, err=%v", &node, &err)
dir, err := d.Dir.Mkdir(req.Name)
if err != nil {
return nil, translateError(err)
}
return &Dir{dir}, nil
}
var _ fusefs.NodeRemover = (*Dir)(nil)
// Remove removes the entry with the given name from
// the receiver, which must be a directory. The entry to be removed
// may correspond to a file (unlink) or to a directory (rmdir).
func (d *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) (err error) {
defer log.Trace(d, "name=%q", req.Name)("err=%v", &err)
err = d.Dir.RemoveName(req.Name)
if err != nil {
return translateError(err)
}
return nil
}
// Check interface satisfied
var _ fusefs.NodeRenamer = (*Dir)(nil)
// Rename the file
func (d *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fusefs.Node) (err error) {
defer log.Trace(d, "oldName=%q, newName=%q, newDir=%+v", req.OldName, req.NewName, newDir)("err=%v", &err)
destDir, ok := newDir.(*Dir)
if !ok {
return errors.Errorf("Unknown Dir type %T", newDir)
}
err = d.Dir.Rename(req.OldName, req.NewName, destDir.Dir)
if err != nil {
return translateError(err)
}
return nil
}
// Check interface satisfied
var _ fusefs.NodeFsyncer = (*Dir)(nil)
// Fsync the directory
func (d *Dir) Fsync(ctx context.Context, req *fuse.FsyncRequest) (err error) {
defer log.Trace(d, "")("err=%v", &err)
err = d.Dir.Sync()
if err != nil {
return translateError(err)
}
return nil
}
// Check interface satisfied
var _ fusefs.NodeLinker = (*Dir)(nil)
// Link creates a new directory entry in the receiver based on an
// existing Node. Receiver must be a directory.
func (d *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fusefs.Node) (newNode fusefs.Node, err error) {
defer log.Trace(d, "req=%v, old=%v", req, old)("new=%v, err=%v", &newNode, &err)
return nil, fuse.ENOSYS
}

128
.rclone_repo/cmd/mount/file.go Executable file
View File

@@ -0,0 +1,128 @@
// +build linux darwin freebsd
package mount
import (
"io"
"time"
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs/log"
"github.com/ncw/rclone/vfs"
"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
)
// File represents a file
type File struct {
*vfs.File
}
// Check interface satisfied
var _ fusefs.Node = (*File)(nil)
// Attr fills out the attributes for the file
func (f *File) Attr(ctx context.Context, a *fuse.Attr) (err error) {
defer log.Trace(f, "")("a=%+v, err=%v", a, &err)
a.Valid = mountlib.AttrTimeout
modTime := f.File.ModTime()
Size := uint64(f.File.Size())
Blocks := (Size + 511) / 512
a.Gid = f.VFS().Opt.GID
a.Uid = f.VFS().Opt.UID
a.Mode = f.VFS().Opt.FilePerms
a.Size = Size
a.Atime = modTime
a.Mtime = modTime
a.Ctime = modTime
a.Crtime = modTime
a.Blocks = Blocks
return nil
}
// Check interface satisfied
var _ fusefs.NodeSetattrer = (*File)(nil)
// Setattr handles attribute changes from FUSE. Currently supports ModTime and Size only
func (f *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) (err error) {
defer log.Trace(f, "a=%+v", req)("err=%v", &err)
if !f.VFS().Opt.NoModTime {
if req.Valid.Mtime() {
err = f.File.SetModTime(req.Mtime)
} else if req.Valid.MtimeNow() {
err = f.File.SetModTime(time.Now())
}
}
if req.Valid.Size() {
err = f.File.Truncate(int64(req.Size))
}
return translateError(err)
}
// Check interface satisfied
var _ fusefs.NodeOpener = (*File)(nil)
// Open the file for read or write
func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fh fusefs.Handle, err error) {
defer log.Trace(f, "flags=%v", req.Flags)("fh=%v, err=%v", &fh, &err)
// fuse flags are based off syscall flags as are os flags, so
// should be compatible
handle, err := f.File.Open(int(req.Flags))
if err != nil {
return nil, translateError(err)
}
// See if seeking is supported and set FUSE hint accordingly
if _, err = handle.Seek(0, io.SeekCurrent); err != nil {
resp.Flags |= fuse.OpenNonSeekable
}
return &FileHandle{handle}, nil
}
// Check interface satisfied
var _ fusefs.NodeFsyncer = (*File)(nil)
// Fsync the file
//
// Note that we don't do anything except return OK
func (f *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) (err error) {
defer log.Trace(f, "")("err=%v", &err)
return nil
}
// Getxattr gets an extended attribute by the given name from the
// node.
//
// If there is no xattr by that name, returns fuse.ErrNoXattr.
func (f *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
return fuse.ENOSYS // we never implement this
}
var _ fusefs.NodeGetxattrer = (*File)(nil)
// Listxattr lists the extended attributes recorded for the node.
func (f *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
return fuse.ENOSYS // we never implement this
}
var _ fusefs.NodeListxattrer = (*File)(nil)
// Setxattr sets an extended attribute with the given name and
// value for the node.
func (f *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
return fuse.ENOSYS // we never implement this
}
var _ fusefs.NodeSetxattrer = (*File)(nil)
// Removexattr removes an extended attribute for the name.
//
// If there is no xattr by that name, returns fuse.ErrNoXattr.
func (f *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {
return fuse.ENOSYS // we never implement this
}
var _ fusefs.NodeRemovexattrer = (*File)(nil)

112
.rclone_repo/cmd/mount/fs.go Executable file
View File

@@ -0,0 +1,112 @@
// FUSE main Fs
// +build linux darwin freebsd
package mount
import (
"syscall"
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/log"
"github.com/ncw/rclone/vfs"
"github.com/ncw/rclone/vfs/vfsflags"
"github.com/pkg/errors"
"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
)
// FS represents the top level filing system
type FS struct {
*vfs.VFS
f fs.Fs
}
// Check interface satistfied
var _ fusefs.FS = (*FS)(nil)
// NewFS makes a new FS
func NewFS(f fs.Fs) *FS {
fsys := &FS{
VFS: vfs.New(f, &vfsflags.Opt),
f: f,
}
return fsys
}
// Root returns the root node
func (f *FS) Root() (node fusefs.Node, err error) {
defer log.Trace("", "")("node=%+v, err=%v", &node, &err)
root, err := f.VFS.Root()
if err != nil {
return nil, translateError(err)
}
return &Dir{root}, nil
}
// Check interface satsified
var _ fusefs.FSStatfser = (*FS)(nil)
// Statfs is called to obtain file system metadata.
// It should write that data to resp.
func (f *FS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) (err error) {
defer log.Trace("", "")("stat=%+v, err=%v", resp, &err)
const blockSize = 4096
const fsBlocks = (1 << 50) / blockSize
resp.Blocks = fsBlocks // Total data blocks in file system.
resp.Bfree = fsBlocks // Free blocks in file system.
resp.Bavail = fsBlocks // Free blocks in file system if you're not root.
resp.Files = 1E9 // Total files in file system.
resp.Ffree = 1E9 // Free files in file system.
resp.Bsize = blockSize // Block size
resp.Namelen = 255 // Maximum file name length?
resp.Frsize = blockSize // Fragment size, smallest addressable data size in the file system.
total, used, free := f.VFS.Statfs()
if total >= 0 {
resp.Blocks = uint64(total) / blockSize
}
if used >= 0 {
resp.Bfree = resp.Blocks - uint64(used)/blockSize
}
if free >= 0 {
resp.Bavail = uint64(free) / blockSize
}
mountlib.ClipBlocks(&resp.Blocks)
mountlib.ClipBlocks(&resp.Bfree)
mountlib.ClipBlocks(&resp.Bavail)
return nil
}
// Translate errors from mountlib
func translateError(err error) error {
if err == nil {
return nil
}
switch errors.Cause(err) {
case vfs.OK:
return nil
case vfs.ENOENT:
return fuse.ENOENT
case vfs.EEXIST:
return fuse.EEXIST
case vfs.EPERM:
return fuse.EPERM
case vfs.ECLOSED:
return fuse.Errno(syscall.EBADF)
case vfs.ENOTEMPTY:
return fuse.Errno(syscall.ENOTEMPTY)
case vfs.ESPIPE:
return fuse.Errno(syscall.ESPIPE)
case vfs.EBADF:
return fuse.Errno(syscall.EBADF)
case vfs.EROFS:
return fuse.Errno(syscall.EROFS)
case vfs.ENOSYS:
return fuse.ENOSYS
case vfs.EINVAL:
return fuse.Errno(syscall.EINVAL)
}
return err
}

View File

@@ -0,0 +1,84 @@
// +build linux darwin freebsd
package mount
import (
"io"
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/fs/log"
"github.com/ncw/rclone/vfs"
"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
)
// FileHandle is an open for read file handle on a File
type FileHandle struct {
vfs.Handle
}
// Check interface satisfied
var _ fusefs.HandleReader = (*FileHandle)(nil)
// Read from the file handle
func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) (err error) {
var n int
defer log.Trace(fh, "len=%d, offset=%d", req.Size, req.Offset)("read=%d, err=%v", &n, &err)
data := make([]byte, req.Size)
n, err = fh.Handle.ReadAt(data, req.Offset)
if err == io.EOF {
err = nil
} else if err != nil {
return translateError(err)
}
resp.Data = data[:n]
return nil
}
// Check interface satisfied
var _ fusefs.HandleWriter = (*FileHandle)(nil)
// Write data to the file handle
func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) (err error) {
defer log.Trace(fh, "len=%d, offset=%d", len(req.Data), req.Offset)("written=%d, err=%v", &resp.Size, &err)
n, err := fh.Handle.WriteAt(req.Data, req.Offset)
if err != nil {
return translateError(err)
}
resp.Size = int(n)
return nil
}
// Check interface satisfied
var _ fusefs.HandleFlusher = (*FileHandle)(nil)
// Flush is called on each close() of a file descriptor. So if a
// filesystem wants to return write errors in close() and the file has
// cached dirty data, this is a good place to write back data and
// return any errors. Since many applications ignore close() errors
// this is not always useful.
//
// NOTE: The flush() method may be called more than once for each
// open(). This happens if more than one file descriptor refers to an
// opened file due to dup(), dup2() or fork() calls. It is not
// possible to determine if a flush is final, so each flush should be
// treated equally. Multiple write-flush sequences are relatively
// rare, so this shouldn't be a problem.
//
// Filesystems shouldn't assume that flush will always be called after
// some writes, or that if will be called at all.
func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) (err error) {
defer log.Trace(fh, "")("err=%v", &err)
return translateError(fh.Handle.Flush())
}
var _ fusefs.HandleReleaser = (*FileHandle)(nil)
// Release is called when we are finished with the file handle
//
// It isn't called directly from userspace so the error is ignored by
// the kernel
func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) (err error) {
defer log.Trace(fh, "")("err=%v", &err)
return translateError(fh.Handle.Release())
}

174
.rclone_repo/cmd/mount/mount.go Executable file
View File

@@ -0,0 +1,174 @@
// Package mount implents a FUSE mounting system for rclone remotes.
// +build linux darwin freebsd
package mount
import (
"fmt"
"os"
"os/signal"
"syscall"
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/lib/atexit"
"github.com/ncw/rclone/vfs"
"github.com/ncw/rclone/vfs/vfsflags"
"github.com/okzk/sdnotify"
"github.com/pkg/errors"
)
func init() {
mountlib.NewMountCommand("mount", Mount)
}
// mountOptions configures the options from the command line flags
func mountOptions(device string) (options []fuse.MountOption) {
options = []fuse.MountOption{
fuse.MaxReadahead(uint32(mountlib.MaxReadAhead)),
fuse.Subtype("rclone"),
fuse.FSName(device),
fuse.VolumeName(mountlib.VolumeName),
// Options from benchmarking in the fuse module
//fuse.MaxReadahead(64 * 1024 * 1024),
//fuse.AsyncRead(), - FIXME this causes
// ReadFileHandle.Read error: read /home/files/ISOs/xubuntu-15.10-desktop-amd64.iso: bad file descriptor
// which is probably related to errors people are having
//fuse.WritebackCache(),
}
if mountlib.NoAppleDouble {
options = append(options, fuse.NoAppleDouble())
}
if mountlib.NoAppleXattr {
options = append(options, fuse.NoAppleXattr())
}
if mountlib.AllowNonEmpty {
options = append(options, fuse.AllowNonEmptyMount())
}
if mountlib.AllowOther {
options = append(options, fuse.AllowOther())
}
if mountlib.AllowRoot {
options = append(options, fuse.AllowRoot())
}
if mountlib.DefaultPermissions {
options = append(options, fuse.DefaultPermissions())
}
if vfsflags.Opt.ReadOnly {
options = append(options, fuse.ReadOnly())
}
if mountlib.WritebackCache {
options = append(options, fuse.WritebackCache())
}
if mountlib.DaemonTimeout != 0 {
options = append(options, fuse.DaemonTimeout(fmt.Sprint(int(mountlib.DaemonTimeout.Seconds()))))
}
if len(mountlib.ExtraOptions) > 0 {
fs.Errorf(nil, "-o/--option not supported with this FUSE backend")
}
if len(mountlib.ExtraOptions) > 0 {
fs.Errorf(nil, "--fuse-flag not supported with this FUSE backend")
}
return options
}
// mount the file system
//
// The mount point will be ready when this returns.
//
// returns an error, and an error channel for the serve process to
// report an error when fusermount is called.
func mount(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error) {
fs.Debugf(f, "Mounting on %q", mountpoint)
c, err := fuse.Mount(mountpoint, mountOptions(f.Name()+":"+f.Root())...)
if err != nil {
return nil, nil, nil, err
}
filesys := NewFS(f)
server := fusefs.New(c, nil)
// Serve the mount point in the background returning error to errChan
errChan := make(chan error, 1)
go func() {
err := server.Serve(filesys)
closeErr := c.Close()
if err == nil {
err = closeErr
}
errChan <- err
}()
// check if the mount process has an error to report
<-c.Ready
if err := c.MountError; err != nil {
return nil, nil, nil, err
}
unmount := func() error {
// Shutdown the VFS
filesys.VFS.Shutdown()
return fuse.Unmount(mountpoint)
}
return filesys.VFS, errChan, unmount, nil
}
// Mount mounts the remote at mountpoint.
//
// If noModTime is set then it
func Mount(f fs.Fs, mountpoint string) error {
if mountlib.DebugFUSE {
fuse.Debug = func(msg interface{}) {
fs.Debugf("fuse", "%v", msg)
}
}
// Mount it
FS, errChan, unmount, err := mount(f, mountpoint)
if err != nil {
return errors.Wrap(err, "failed to mount FUSE fs")
}
sigInt := make(chan os.Signal, 1)
signal.Notify(sigInt, syscall.SIGINT, syscall.SIGTERM)
sigHup := make(chan os.Signal, 1)
signal.Notify(sigHup, syscall.SIGHUP)
atexit.IgnoreSignals()
if err := sdnotify.Ready(); err != nil && err != sdnotify.ErrSdNotifyNoSocket {
return errors.Wrap(err, "failed to notify systemd")
}
waitloop:
for {
select {
// umount triggered outside the app
case err = <-errChan:
break waitloop
// Program abort: umount
case <-sigInt:
err = unmount()
break waitloop
// user sent SIGHUP to clear the cache
case <-sigHup:
root, err := FS.Root()
if err != nil {
fs.Errorf(f, "Error reading root: %v", err)
} else {
root.ForgetAll()
}
}
}
_ = sdnotify.Stopping()
if err != nil {
return errors.Wrap(err, "failed to umount FUSE fs")
}
return nil
}

View File

@@ -0,0 +1,13 @@
// +build linux darwin freebsd
package mount
import (
"testing"
"github.com/ncw/rclone/cmd/mountlib/mounttest"
)
func TestMount(t *testing.T) {
mounttest.RunTests(t, mount)
}

View File

@@ -0,0 +1,6 @@
// Build for mount for unsupported platforms to stop go complaining
// about "no buildable Go source files "
// +build !linux,!darwin,!freebsd
package mount

View File

@@ -0,0 +1,72 @@
// +build ignore
// Read blocks out of a single file to time the seeking code
package main
import (
"flag"
"io"
"log"
"math/rand"
"os"
"time"
)
var (
// Flags
iterations = flag.Int("n", 25, "Iterations to try")
maxBlockSize = flag.Int("b", 1024*1024, "Max block size to read")
randSeed = flag.Int64("seed", 1, "Seed for the random number generator")
)
func randomSeekTest(size int64, in *os.File, name string) {
start := rand.Int63n(size)
blockSize := rand.Intn(*maxBlockSize)
if int64(blockSize) > size-start {
blockSize = int(size - start)
}
log.Printf("Reading %d from %d", blockSize, start)
_, err := in.Seek(start, io.SeekStart)
if err != nil {
log.Fatalf("Seek failed on %q: %v", name, err)
}
buf := make([]byte, blockSize)
_, err = io.ReadFull(in, buf)
if err != nil {
log.Fatalf("Read failed on %q: %v", name, err)
}
}
func main() {
flag.Parse()
args := flag.Args()
if len(args) != 1 {
log.Fatalf("Require 1 file as argument")
}
rand.Seed(*randSeed)
name := args[0]
in, err := os.Open(name)
if err != nil {
log.Fatalf("Couldn't open %q: %v", name, err)
}
fi, err := in.Stat()
if err != nil {
log.Fatalf("Couldn't stat %q: %v", name, err)
}
start := time.Now()
for i := 0; i < *iterations; i++ {
randomSeekTest(fi.Size(), in, name)
}
dt := time.Since(start)
log.Printf("That took %v for %d iterations, %v per iteration", dt, *iterations, dt/time.Duration(*iterations))
err = in.Close()
if err != nil {
log.Fatalf("Error closing %q: %v", name, err)
}
}

View File

@@ -0,0 +1,115 @@
// +build ignore
// Read two files with lots of seeking to stress test the seek code
package main
import (
"bytes"
"flag"
"io"
"io/ioutil"
"log"
"math/rand"
"os"
"time"
)
var (
// Flags
iterations = flag.Int("n", 1E6, "Iterations to try")
maxBlockSize = flag.Int("b", 1024*1024, "Max block size to read")
)
func init() {
rand.Seed(time.Now().UnixNano())
}
func randomSeekTest(size int64, in1, in2 *os.File, file1, file2 string) {
start := rand.Int63n(size)
blockSize := rand.Intn(*maxBlockSize)
if int64(blockSize) > size-start {
blockSize = int(size - start)
}
log.Printf("Reading %d from %d", blockSize, start)
_, err := in1.Seek(start, io.SeekStart)
if err != nil {
log.Fatalf("Seek failed on %q: %v", file1, err)
}
_, err = in2.Seek(start, io.SeekStart)
if err != nil {
log.Fatalf("Seek failed on %q: %v", file2, err)
}
buf1 := make([]byte, blockSize)
n1, err := io.ReadFull(in1, buf1)
if err != nil {
log.Fatalf("Read failed on %q: %v", file1, err)
}
buf2 := make([]byte, blockSize)
n2, err := io.ReadFull(in2, buf2)
if err != nil {
log.Fatalf("Read failed on %q: %v", file2, err)
}
if n1 != n2 {
log.Fatalf("Read different lengths %d (%q) != %d (%q)", n1, file1, n2, file2)
}
if !bytes.Equal(buf1, buf2) {
log.Printf("Dumping different blocks")
err = ioutil.WriteFile("/tmp/z1", buf1, 0777)
if err != nil {
log.Fatalf("Failed to write /tmp/z1: %v", err)
}
err = ioutil.WriteFile("/tmp/z2", buf2, 0777)
if err != nil {
log.Fatalf("Failed to write /tmp/z2: %v", err)
}
log.Fatalf("Read different contents - saved in /tmp/z1 and /tmp/z2")
}
}
func main() {
flag.Parse()
args := flag.Args()
if len(args) != 2 {
log.Fatalf("Require 2 files as argument")
}
file1, file2 := args[0], args[1]
in1, err := os.Open(file1)
if err != nil {
log.Fatalf("Couldn't open %q: %v", file1, err)
}
in2, err := os.Open(file2)
if err != nil {
log.Fatalf("Couldn't open %q: %v", file2, err)
}
fi1, err := in1.Stat()
if err != nil {
log.Fatalf("Couldn't stat %q: %v", file1, err)
}
fi2, err := in2.Stat()
if err != nil {
log.Fatalf("Couldn't stat %q: %v", file2, err)
}
if fi1.Size() != fi2.Size() {
log.Fatalf("Files not the same size")
}
for i := 0; i < *iterations; i++ {
randomSeekTest(fi1.Size(), in1, in2, file1, file2)
}
err = in1.Close()
if err != nil {
log.Fatalf("Error closing %q: %v", file1, err)
}
err = in2.Close()
if err != nil {
log.Fatalf("Error closing %q: %v", file2, err)
}
}

View File

@@ -0,0 +1,129 @@
// +build ignore
// Read lots files with lots of simultaneous seeking to stress test the seek code
package main
import (
"flag"
"io"
"log"
"math/rand"
"os"
"path/filepath"
"sort"
"sync"
"time"
)
var (
// Flags
iterations = flag.Int("n", 1E6, "Iterations to try")
maxBlockSize = flag.Int("b", 1024*1024, "Max block size to read")
simultaneous = flag.Int("transfers", 16, "Number of simultaneous files to open")
seeksPerFile = flag.Int("seeks", 8, "Seeks per file")
mask = flag.Int64("mask", 0, "mask for seek, eg 0x7fff")
)
func init() {
rand.Seed(time.Now().UnixNano())
}
func seekTest(n int, file string) {
in, err := os.Open(file)
if err != nil {
log.Fatalf("Couldn't open %q: %v", file, err)
}
fi, err := in.Stat()
if err != nil {
log.Fatalf("Couldn't stat %q: %v", file, err)
}
size := fi.Size()
// FIXME make sure we try start and end
maxBlockSize := *maxBlockSize
if int64(maxBlockSize) > size {
maxBlockSize = int(size)
}
for i := 0; i < n; i++ {
start := rand.Int63n(size)
if *mask != 0 {
start &^= *mask
}
blockSize := rand.Intn(maxBlockSize)
beyondEnd := false
switch rand.Intn(10) {
case 0:
start = 0
case 1:
start = size - int64(blockSize)
case 2:
// seek beyond the end
start = size + int64(blockSize)
beyondEnd = true
default:
}
if !beyondEnd && int64(blockSize) > size-start {
blockSize = int(size - start)
}
log.Printf("%s: Reading %d from %d", file, blockSize, start)
_, err = in.Seek(start, io.SeekStart)
if err != nil {
log.Fatalf("Seek failed on %q: %v", file, err)
}
buf := make([]byte, blockSize)
n, err := io.ReadFull(in, buf)
if beyondEnd && err == io.EOF {
// OK
} else if err != nil {
log.Fatalf("Read failed on %q: %v (%d)", file, err, n)
}
}
err = in.Close()
if err != nil {
log.Fatalf("Error closing %q: %v", file, err)
}
}
// Find all the files in dir
func findFiles(dir string) (files []string) {
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if info.Mode().IsRegular() && info.Size() > 0 {
files = append(files, path)
}
return nil
})
sort.Strings(files)
return files
}
func main() {
flag.Parse()
args := flag.Args()
if len(args) != 1 {
log.Fatalf("Require a directory as argument")
}
dir := args[0]
files := findFiles(dir)
jobs := make(chan string, *simultaneous)
var wg sync.WaitGroup
wg.Add(*simultaneous)
for i := 0; i < *simultaneous; i++ {
go func() {
defer wg.Done()
for file := range jobs {
seekTest(*seeksPerFile, file)
}
}()
}
for i := 0; i < *iterations; i++ {
i := rand.Intn(len(files))
jobs <- files[i]
//jobs <- files[i]
}
close(jobs)
wg.Wait()
}