commit c8eb52f9ba4f00bf8a75c1ca52131cae52bc3eb4 Author: bel Date: Mon Jan 13 03:37:51 2020 +0000 overdue diff --git a/.rclone_repo/.appveyor.yml b/.rclone_repo/.appveyor.yml new file mode 100755 index 0000000..c9afa18 --- /dev/null +++ b/.rclone_repo/.appveyor.yml @@ -0,0 +1,46 @@ +version: "{build}" + +os: Windows Server 2012 R2 + +clone_folder: c:\gopath\src\github.com\ncw\rclone + +environment: + GOPATH: C:\gopath + CPATH: C:\Program Files (x86)\WinFsp\inc\fuse + ORIGPATH: '%PATH%' + NOCCPATH: C:\MinGW\bin;%GOPATH%\bin;%PATH% + PATHCC64: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin;%NOCCPATH% + PATHCC32: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%NOCCPATH% + PATH: '%PATHCC64%' + RCLONE_CONFIG_PASS: + secure: HbzxSy9zQ8NYWN9NNPf6ALQO9Q0mwRNqwehsLcOEHy0= + +install: +- choco install winfsp -y +- choco install zip -y +- copy c:\MinGW\bin\mingw32-make.exe c:\MinGW\bin\make.exe + +build_script: +- echo %PATH% +- echo %GOPATH% +- go version +- go env +- go install +- go build +- make log_since_last_release > %TEMP%\git-log.txt +- make version > %TEMP%\version +- set /p RCLONE_VERSION=<%TEMP%\version +- set PATH=%PATHCC32% +- go run bin/cross-compile.go -release beta-latest -git-log %TEMP%\git-log.txt -include "^windows/386" -cgo -tags cmount %RCLONE_VERSION% +- set PATH=%PATHCC64% +- go run bin/cross-compile.go -release beta-latest -git-log %TEMP%\git-log.txt -include "^windows/amd64" -cgo -no-clean -tags cmount %RCLONE_VERSION% + +test_script: +- make GOTAGS=cmount quicktest + +artifacts: +- path: rclone.exe +- path: build/*-v*.zip + +deploy_script: +- IF "%APPVEYOR_PULL_REQUEST_NUMBER%" == "" make appveyor_upload diff --git a/.rclone_repo/.circleci/config.yml b/.rclone_repo/.circleci/config.yml new file mode 100755 index 0000000..edbfa36 --- /dev/null +++ b/.rclone_repo/.circleci/config.yml @@ -0,0 +1,34 @@ +version: 2 + +jobs: + + build: + machine: true + + working_directory: ~/.go_workspace/src/github.com/ncw/rclone + + steps: + - checkout + + - run: + name: Cross-compile rclone + command: | + docker pull billziss/xgo-cgofuse + go get -v github.com/karalabe/xgo + xgo \ + --image=billziss/xgo-cgofuse \ + --targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \ + -tags cmount \ + . + xgo \ + --targets=android/*,ios/* \ + . + + - run: + name: Prepare artifacts + command: | + mkdir -p /tmp/rclone.dist + cp -R rclone-* /tmp/rclone.dist + + - store_artifacts: + path: /tmp/rclone.dist diff --git a/.rclone_repo/.gitignore b/.rclone_repo/.gitignore new file mode 100755 index 0000000..cde1c88 --- /dev/null +++ b/.rclone_repo/.gitignore @@ -0,0 +1,7 @@ +*~ +_junk/ +rclone +build +docs/public +rclone.iml +.idea diff --git a/.rclone_repo/.gometalinter.json b/.rclone_repo/.gometalinter.json new file mode 100755 index 0000000..4cd25d5 --- /dev/null +++ b/.rclone_repo/.gometalinter.json @@ -0,0 +1,14 @@ +{ + "Enable": [ + "deadcode", + "errcheck", + "goimports", + "golint", + "ineffassign", + "structcheck", + "varcheck", + "vet" + ], + "EnableGC": true, + "Vendor": true +} diff --git a/.rclone_repo/.pkgr.yml b/.rclone_repo/.pkgr.yml new file mode 100755 index 0000000..a546ed1 --- /dev/null +++ b/.rclone_repo/.pkgr.yml @@ -0,0 +1,2 @@ +default_dependencies: false +cli: rclone diff --git a/.rclone_repo/.travis.yml b/.rclone_repo/.travis.yml new file mode 100755 index 0000000..d6e28f5 --- /dev/null +++ b/.rclone_repo/.travis.yml @@ -0,0 +1,51 @@ +language: go +sudo: required +dist: trusty +os: +- linux +go: +- 1.7.x +- 1.8.x +- 1.9.x +- 1.10.x +- 1.11.x +- tip +before_install: +- if [[ $TRAVIS_OS_NAME == linux ]]; then sudo modprobe fuse ; sudo chmod 666 /dev/fuse ; sudo chown root:$USER /etc/fuse.conf ; fi +- if [[ $TRAVIS_OS_NAME == osx ]]; then brew update && brew tap caskroom/cask && brew cask install osxfuse ; fi +install: +- git fetch --unshallow --tags +- make vars +- make build_dep +script: +- make check +- make quicktest +- make compile_all +env: + global: + - GOTAGS=cmount + - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA= + - secure: AMjrMAksDy3QwqGqnvtUg8FL/GNVgNqTqhntLF9HSU0njHhX6YurGGnfKdD9vNHlajPQOewvmBjwNLcDWGn2WObdvmh9Ohep0EmOjZ63kliaRaSSQueSd8y0idfqMQAxep0SObOYbEDVmQh0RCAE9wOVKRaPgw98XvgqWGDq5Tw= + - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk= +addons: + apt: + packages: + - fuse + - libfuse-dev + - rpm + - pkg-config +matrix: + allow_failures: + - go: tip + include: + - os: osx + go: 1.11.x + env: GOTAGS="" +deploy: + provider: script + script: make travis_beta + skip_cleanup: true + on: + all_branches: true + go: 1.11.x + condition: $TRAVIS_PULL_REQUEST == false diff --git a/.rclone_repo/CONTRIBUTING.md b/.rclone_repo/CONTRIBUTING.md new file mode 100755 index 0000000..a1acf9c --- /dev/null +++ b/.rclone_repo/CONTRIBUTING.md @@ -0,0 +1,351 @@ +# Contributing to rclone # + +This is a short guide on how to contribute things to rclone. + +## Reporting a bug ## + +If you've just got a question or aren't sure if you've found a bug +then please use the [rclone forum](https://forum.rclone.org/) instead +of filing an issue. + +When filing an issue, please include the following information if +possible as well as a description of the problem. Make sure you test +with the [latest beta of rclone](https://beta.rclone.org/): + + * Rclone version (eg output from `rclone -V`) + * Which OS you are using and how many bits (eg Windows 7, 64 bit) + * The command you were trying to run (eg `rclone copy /tmp remote:tmp`) + * A log of the command with the `-vv` flag (eg output from `rclone -vv copy /tmp remote:tmp`) + * if the log contains secrets then edit the file with a text editor first to obscure them + +## Submitting a pull request ## + +If you find a bug that you'd like to fix, or a new feature that you'd +like to implement then please submit a pull request via Github. + +If it is a big feature then make an issue first so it can be discussed. + +You'll need a Go environment set up with GOPATH set. See [the Go +getting started docs](https://golang.org/doc/install) for more info. + +First in your web browser press the fork button on [rclone's Github +page](https://github.com/ncw/rclone). + +Now in your terminal + + go get -u github.com/ncw/rclone + cd $GOPATH/src/github.com/ncw/rclone + git remote rename origin upstream + git remote add origin git@github.com:YOURUSER/rclone.git + +Make a branch to add your new feature + + git checkout -b my-new-feature + +And get hacking. + +When ready - run the unit tests for the code you changed + + go test -v + +Note that you may need to make a test remote, eg `TestSwift` for some +of the unit tests. + +Note the top level Makefile targets + + * make check + * make test + +Both of these will be run by Travis when you make a pull request but +you can do this yourself locally too. These require some extra go +packages which you can install with + + * make build_dep + +Make sure you + + * Add documentation for a new feature (see below for where) + * Add unit tests for a new feature + * squash commits down to one per feature + * rebase to master `git rebase master` + +When you are done with that + + git push origin my-new-feature + +Go to the Github website and click [Create pull +request](https://help.github.com/articles/creating-a-pull-request/). + +You patch will get reviewed and you might get asked to fix some stuff. + +If so, then make the changes in the same branch, squash the commits, +rebase it to master then push it to Github with `--force`. + +## Testing ## + +rclone's tests are run from the go testing framework, so at the top +level you can run this to run all the tests. + + go test -v ./... + +rclone contains a mixture of unit tests and integration tests. +Because it is difficult (and in some respects pointless) to test cloud +storage systems by mocking all their interfaces, rclone unit tests can +run against any of the backends. This is done by making specially +named remotes in the default config file. + +If you wanted to test changes in the `drive` backend, then you would +need to make a remote called `TestDrive`. + +You can then run the unit tests in the drive directory. These tests +are skipped if `TestDrive:` isn't defined. + + cd backend/drive + go test -v + +You can then run the integration tests which tests all of rclone's +operations. Normally these get run against the local filing system, +but they can be run against any of the remotes. + + cd fs/sync + go test -v -remote TestDrive: + go test -v -remote TestDrive: -subdir + + cd fs/operations + go test -v -remote TestDrive: + +If you want to run all the integration tests against all the remotes, +then change into the project root and run + + make test + +This command is run daily on the the integration test server. You can +find the results at https://pub.rclone.org/integration-tests/ + +## Code Organisation ## + +Rclone code is organised into a small number of top level directories +with modules beneath. + + * backend - the rclone backends for interfacing to cloud providers - + * all - import this to load all the cloud providers + * ...providers + * bin - scripts for use while building or maintaining rclone + * cmd - the rclone commands + * all - import this to load all the commands + * ...commands + * docs - the documentation and website + * content - adjust these docs only - everything else is autogenerated + * fs - main rclone definitions - minimal amount of code + * accounting - bandwidth limiting and statistics + * asyncreader - an io.Reader which reads ahead + * config - manage the config file and flags + * driveletter - detect if a name is a drive letter + * filter - implements include/exclude filtering + * fserrors - rclone specific error handling + * fshttp - http handling for rclone + * fspath - path handling for rclone + * hash - defines rclones hash types and functions + * list - list a remote + * log - logging facilities + * march - iterates directories in lock step + * object - in memory Fs objects + * operations - primitives for sync, eg Copy, Move + * sync - sync directories + * walk - walk a directory + * fstest - provides integration test framework + * fstests - integration tests for the backends + * mockdir - mocks an fs.Directory + * mockobject - mocks an fs.Object + * test_all - Runs integration tests for everything + * graphics - the images used in the website etc + * lib - libraries used by the backend + * atexit - register functions to run when rclone exits + * dircache - directory ID to name caching + * oauthutil - helpers for using oauth + * pacer - retries with backoff and paces operations + * readers - a selection of useful io.Readers + * rest - a thin abstraction over net/http for REST + * vendor - 3rd party code managed by `go mod` + * vfs - Virtual FileSystem layer for implementing rclone mount and similar + +## Writing Documentation ## + +If you are adding a new feature then please update the documentation. + +If you add a new flag, then if it is a general flag, document it in +`docs/content/docs.md` - the flags there are supposed to be in +alphabetical order. If it is a remote specific flag, then document it +in `docs/content/remote.md`. + +The only documentation you need to edit are the `docs/content/*.md` +files. The MANUAL.*, rclone.1, web site etc are all auto generated +from those during the release process. See the `make doc` and `make +website` targets in the Makefile if you are interested in how. You +don't need to run these when adding a feature. + +Documentation for rclone sub commands is with their code, eg +`cmd/ls/ls.go`. + +## Making a release ## + +There are separate instructions for making a release in the RELEASE.md +file. + +## Commit messages ## + +Please make the first line of your commit message a summary of the +change, and prefix it with the directory of the change followed by a +colon. The changelog gets made by looking at just these first lines +so make it good! + +If you have more to say about the commit, then enter a blank line and +carry on the description. Remember to say why the change was needed - +the commit itself shows what was changed. + +If the change fixes an issue then write `Fixes #1234` in the commit +message. This can be on the subject line if it will fit. If you +don't want to close the associated issue just put `#1234` and the +change will get linked into the issue. + +Here is an example of a short commit message: + +``` +drive: add team drive support - fixes #885 +``` + +And here is an example of a longer one: + +``` +mount: fix hang on errored upload + +In certain circumstances if an upload failed then the mount could hang +indefinitely. This was fixed by closing the read pipe after the Put +completed. This will cause the write side to return a pipe closed +error fixing the hang. + +Fixes #1498 +``` + +## Adding a dependency ## + +rclone uses the [go +modules](https://tip.golang.org/cmd/go/#hdr-Modules__module_versions__and_more) +support in go1.11 and later to manage its dependencies. + +**NB** you must be using go1.11 or above to add a dependency to +rclone. Rclone will still build with older versions of go, but we use +the `go mod` command for dependencies which is only in go1.11 and +above. + +rclone can be built with modules outside of the GOPATH, but for +backwards compatibility with older go versions, rclone also maintains +a `vendor` directory with all the external code rclone needs for +building. + +The `vendor` directory is entirely managed by the `go mod` tool, do +not add things manually. + +To add a dependency `github.com/ncw/new_dependency` see the +instructions below. These will fetch the dependency, add it to +`go.mod` and `go.sum` and vendor it for older go versions. + + export GO111MODULE=on + go get github.com/ncw/new_dependency + go mod vendor + +You can add constraints on that package when doing `go get` (see the +go docs linked above), but don't unless you really need to. + +Please check in the changes generated by `go mod` including the +`vendor` directory and `go.mod` and `go.sum` in a single commit +separate from any other code changes with the title "vendor: add +github.com/ncw/new_dependency". Remember to `git add` any new files +in `vendor`. + +## Updating a dependency ## + +If you need to update a dependency then run + + export GO111MODULE=on + go get -u github.com/pkg/errors + go mod vendor + +Check in in a single commit as above. + +## Updating all the dependencies ## + +In order to update all the dependencies then run `make update`. This +just uses the go modules to update all the modules to their latest +stable release. Check in the changes in a single commit as above. + +This should be done early in the release cycle to pick up new versions +of packages in time for them to get some testing. + +## Updating a backend ## + +If you update a backend then please run the unit tests and the +integration tests for that backend. + +Assuming the backend is called `remote`, make create a config entry +called `TestRemote` for the tests to use. + +Now `cd remote` and run `go test -v` to run the unit tests. + +Then `cd fs` and run `go test -v -remote TestRemote:` to run the +integration tests. + +The next section goes into more detail about the tests. + +## Writing a new backend ## + +Choose a name. The docs here will use `remote` as an example. + +Note that in rclone terminology a file system backend is called a +remote or an fs. + +Research + + * Look at the interfaces defined in `fs/fs.go` + * Study one or more of the existing remotes + +Getting going + + * Create `backend/remote/remote.go` (copy this from a similar remote) + * box is a good one to start from if you have a directory based remote + * b2 is a good one to start from if you have a bucket based remote + * Add your remote to the imports in `backend/all/all.go` + * HTTP based remotes are easiest to maintain if they use rclone's rest module, but if there is a really good go SDK then use that instead. + * Try to implement as many optional methods as possible as it makes the remote more usable. + +Unit tests + + * Create a config entry called `TestRemote` for the unit tests to use + * Create a `backend/remote/remote_test.go` - copy and adjust your example remote + * Make sure all tests pass with `go test -v` + +Integration tests + + * Add your fs to `fstest/test_all/test_all.go` + * Make sure integration tests pass with + * `cd fs/operations` + * `go test -v -remote TestRemote:` + * `cd fs/sync` + * `go test -v -remote TestRemote:` + * If you are making a bucket based remote, then check with this also + * `go test -v -remote TestRemote: -subdir` + * And if your remote defines `ListR` this also + * `go test -v -remote TestRemote: -fast-list` + +See the [testing](#testing) section for more information on integration tests. + +Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](http://fontawesome.io/icons/). Keep lists of remotes in alphabetical order but with the local file system last. + + * `README.md` - main Github page + * `docs/content/remote.md` - main docs page + * `docs/content/overview.md` - overview docs + * `docs/content/docs.md` - list of remotes in config section + * `docs/content/about.md` - front page of rclone.org + * `docs/layouts/chrome/navbar.html` - add it to the website navigation + * `bin/make_manual.py` - add the page to the `docs` constant + * `cmd/cmd.go` - the main help for rclone diff --git a/.rclone_repo/COPYING b/.rclone_repo/COPYING new file mode 100755 index 0000000..8c27c67 --- /dev/null +++ b/.rclone_repo/COPYING @@ -0,0 +1,20 @@ +Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/.rclone_repo/ISSUE_TEMPLATE.md b/.rclone_repo/ISSUE_TEMPLATE.md new file mode 100755 index 0000000..9336d75 --- /dev/null +++ b/.rclone_repo/ISSUE_TEMPLATE.md @@ -0,0 +1,43 @@ + + +#### What is the problem you are having with rclone? + + +#### What is your rclone version (eg output from `rclone -V`) + + +#### Which OS you are using and how many bits (eg Windows 7, 64 bit) + + +#### Which cloud storage system are you using? (eg Google Drive) + + +#### The command you were trying to run (eg `rclone copy /tmp remote:tmp`) + + +#### A log from the command with the `-vv` flag (eg output from `rclone -vv copy /tmp remote:tmp`) + diff --git a/.rclone_repo/MAINTAINERS.md b/.rclone_repo/MAINTAINERS.md new file mode 100755 index 0000000..175a55c --- /dev/null +++ b/.rclone_repo/MAINTAINERS.md @@ -0,0 +1,87 @@ +# Maintainers guide for rclone # + +Current active maintainers of rclone are + + * Nick Craig-Wood @ncw + * Stefan Breunig @breunigs + * Ishuah Kariuki @ishuah + * Remus Bunduc @remusb - cache subsystem maintainer + * Fabian Möller @B4dM4n + +**This is a work in progress Draft** + +This is a guide for how to be an rclone maintainer. This is mostly a writeup of what I (@ncw) attempt to do. + +## Triaging Tickets ## + +When a ticket comes in it should be triaged. This means it should be classified by adding labels and placed into a milestone. Quite a lot of tickets need a bit of back and forth to determine whether it is a valid ticket so tickets may remain without labels or milestone for a while. + +Rclone uses the labels like this: + +* `bug` - a definite verified bug +* `can't reproduce` - a problem which we can't reproduce +* `doc fix` - a bug in the documentation - if users need help understanding the docs add this label +* `duplicate` - normally close these and ask the user to subscribe to the original +* `enhancement: new remote` - a new rclone backend +* `enhancement` - a new feature +* `FUSE` - do do with `rclone mount` command +* `good first issue` - mark these if you find a small self contained issue - these get shown to new visitors to the project +* `help` wanted - mark these if you find a self contained issue - these get shown to new visitors to the project +* `IMPORTANT` - note to maintainers not to forget to fix this for the release +* `maintenance` - internal enhancement, code re-organisation etc +* `Needs Go 1.XX` - waiting for that version of Go to be released +* `question` - not a `bug` or `enhancement` - direct to the forum for next time +* `Remote: XXX` - which rclone backend this affects +* `thinking` - not decided on the course of action yet + +If it turns out to be a bug or an enhancement it should be tagged as such, with the appropriate other tags. Don't forget the "good first issue" tag to give new contributors something easy to do to get going. + +When a ticket is tagged it should be added to a milestone, either the next release, the one after, Soon or Help Wanted. Bugs can be added to the "Known Bugs" milestone if they aren't planned to be fixed or need to wait for something (eg the next go release). + +The milestones have these meanings: + +* v1.XX - stuff we would like to fit into this release +* v1.XX+1 - stuff we are leaving until the next release +* Soon - stuff we think is a good idea - waiting to be scheduled to a release +* Help wanted - blue sky stuff that might get moved up, or someone could help with +* Known bugs - bugs waiting on external factors or we aren't going to fix for the moment + +Tickets [with no milestone](https://github.com/ncw/rclone/issues?utf8=✓&q=is%3Aissue%20is%3Aopen%20no%3Amile) are good candidates for ones that have slipped between the gaps and need following up. + +## Closing Tickets ## + +Close tickets as soon as you can - make sure they are tagged with a release. Post a link to a beta in the ticket with the fix in, asking for feedback. + +## Pull requests ## + +Try to process pull requests promptly! + +Merging pull requests on Github itself works quite well now-a-days so you can squash and rebase or rebase pull requests. rclone doesn't use merge commits. Use the squash and rebase option if you need to edit the commit message. + +After merging the commit, in your local master branch, do `git pull` then run `bin/update-authors.py` to update the authors file then `git push`. + +Sometimes pull requests need to be left open for a while - this especially true of contributions of new backends which take a long time to get right. + +## Merges ## + +If you are merging a branch locally then do `git merge --ff-only branch-name` to avoid a merge commit. You'll need to rebase the branch if it doesn't merge cleanly. + +## Release cycle ## + +Rclone aims for a 6-8 week release cycle. Sometimes release cycles take longer if there is something big to merge that didn't stabilize properly or for personal reasons. + +High impact regressions should be fixed before the next release. + +Near the start of the release cycle the dependencies should be updated with `make update` to give time for bugs to surface. + +Towards the end of the release cycle try not to merge anything too big so let things settle down. + +Follow the instructions in RELEASE.md for making the release. Note that the testing part is the most time consuming often needing several rounds of test and fix depending on exactly how many new features rclone has gained. + +## Mailing list ## + +There is now an invite only mailing list for rclone developers `rclone-dev` on google groups. + +## TODO ## + +I should probably make a dev@rclone.org to register with cloud providers. diff --git a/.rclone_repo/MANUAL.html b/.rclone_repo/MANUAL.html new file mode 100755 index 0000000..ec57ccc --- /dev/null +++ b/.rclone_repo/MANUAL.html @@ -0,0 +1,9218 @@ + + + + + + + + rclone(1) User Manual + + + + +

Rclone

+

Logo

+

Rclone is a command line program to sync files and directories to and from:

+ +

Features

+ +

Links

+ +

Install

+

Rclone is a Go program and comes as a single binary file.

+

Quickstart

+ +

See below for some expanded Linux / macOS instructions.

+

See the Usage section of the docs for how to use rclone, or run rclone -h.

+

Script installation

+

To install rclone on Linux/macOS/BSD systems, run:

+
curl https://rclone.org/install.sh | sudo bash
+

For beta installation, run:

+
curl https://rclone.org/install.sh | sudo bash -s beta
+

Note that this script checks the version of rclone installed first and won't re-download if not needed.

+

Linux installation from precompiled binary

+

Fetch and unpack

+
curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip
+unzip rclone-current-linux-amd64.zip
+cd rclone-*-linux-amd64
+

Copy binary file

+
sudo cp rclone /usr/bin/
+sudo chown root:root /usr/bin/rclone
+sudo chmod 755 /usr/bin/rclone
+

Install manpage

+
sudo mkdir -p /usr/local/share/man/man1
+sudo cp rclone.1 /usr/local/share/man/man1/
+sudo mandb
+

Run rclone config to setup. See rclone config docs for more details.

+
rclone config
+

macOS installation from precompiled binary

+

Download the latest version of rclone.

+
cd && curl -O https://downloads.rclone.org/rclone-current-osx-amd64.zip
+

Unzip the download and cd to the extracted folder.

+
unzip -a rclone-current-osx-amd64.zip && cd rclone-*-osx-amd64
+

Move rclone to your $PATH. You will be prompted for your password.

+
sudo mkdir -p /usr/local/bin
+sudo mv rclone /usr/local/bin/
+

(the mkdir command is safe to run, even if the directory already exists).

+

Remove the leftover files.

+
cd .. && rm -rf rclone-*-osx-amd64 rclone-current-osx-amd64.zip
+

Run rclone config to setup. See rclone config docs for more details.

+
rclone config
+

Install from source

+

Make sure you have at least Go 1.7 installed. Download go if necessary. The latest release is recommended. Then

+
git clone https://github.com/ncw/rclone.git
+cd rclone
+go build
+./rclone version
+

You can also build and install rclone in the GOPATH (which defaults to ~/go) with:

+
go get -u -v github.com/ncw/rclone
+

and this will build the binary in $GOPATH/bin (~/go/bin/rclone by default) after downloading the source to $GOPATH/src/github.com/ncw/rclone (~/go/src/github.com/ncw/rclone by default).

+

Installation with Ansible

+

This can be done with Stefan Weichinger's ansible role.

+

Instructions

+
    +
  1. git clone https://github.com/stefangweichinger/ansible-rclone.git into your local roles-directory
    2. +
  2. add the role to the hosts you want rclone installed to:
    3. +
+
    - hosts: rclone-hosts
+      roles:
+          - rclone
+

Configure

+

First, you'll need to configure rclone. As the object storage systems have quite complicated authentication these are kept in a config file. (See the --config entry for how to find the config file and choose its location.)

+

The easiest way to make the config is to run rclone with the config option:

+
rclone config
+

See the following for detailed instructions for

+ +

Usage

+

Rclone syncs a directory tree from one storage system to another.

+

Its syntax is like this

+
Syntax: [options] subcommand <parameters> <parameters...>
+

Source and destination paths are specified by the name you gave the storage system in the config file then the sub path, eg "drive:myfolder" to look at "myfolder" in Google drive.

+

You can define as many storage paths as you like in the config file.

+

Subcommands

+

rclone uses a system of subcommands. For example

+
rclone ls remote:path # lists a re
+rclone copy /local/path remote:path # copies /local/path to the remote
+rclone sync /local/path remote:path # syncs /local/path to the remote
+

rclone config

+

Enter an interactive configuration session.

+

Synopsis

+

Enter an interactive configuration session where you can setup new remotes and manage existing ones. You may also set or remove a password to protect your configuration.

+
rclone config [flags]
+

Options

+
  -h, --help   help for config
+

rclone copy

+

Copy files from source to dest, skipping already copied

+

Synopsis

+

Copy the source to the destination. Doesn't transfer unchanged files, testing by size and modification time or MD5SUM. Doesn't delete files from the destination.

+

Note that it is always the contents of the directory that is synced, not the directory so when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents.

+

If dest:path doesn't exist, it is created and the source:path contents go there.

+

For example

+
rclone copy source:sourcepath dest:destpath
+

Let's say there are two files in sourcepath

+
sourcepath/one.txt
+sourcepath/two.txt
+

This copies them to

+
destpath/one.txt
+destpath/two.txt
+

Not to

+
destpath/sourcepath/one.txt
+destpath/sourcepath/two.txt
+

If you are familiar with rsync, rclone always works as if you had written a trailing / - meaning "copy the contents of this directory". This applies to all commands and whether you are talking about the source or destination.

+
rclone copy source:path dest:path [flags]
+

Options

+
  -h, --help   help for copy
+

rclone sync

+

Make source and dest identical, modifying destination only.

+

Synopsis

+

Sync the source to the destination, changing the destination only. Doesn't transfer unchanged files, testing by size and modification time or MD5SUM. Destination is updated to match source, including deleting files if necessary.

+

Important: Since this can cause data loss, test first with the --dry-run flag to see exactly what would be copied and deleted.

+

Note that files in the destination won't be deleted if there were any errors at any point.

+

It is always the contents of the directory that is synced, not the directory so when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. See extended explanation in the copy command above if unsure.

+

If dest:path doesn't exist, it is created and the source:path contents go there.

+
rclone sync source:path dest:path [flags]
+

Options

+
  -h, --help   help for sync
+

rclone move

+

Move files from source to dest.

+

Synopsis

+

Moves the contents of the source directory to the destination directory. Rclone will error if the source and destination overlap and the remote does not support a server side directory move operation.

+

If no filters are in use and if possible this will server side move source:path into dest:path. After this source:path will no longer longer exist.

+

Otherwise for each file in source:path selected by the filters (if any) this will move it into dest:path. If possible a server side move will be used, otherwise it will copy it (server side if possible) into dest:path then delete the original (if no errors on copy) in source:path.

+

If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag.

+

Important: Since this can cause data loss, test first with the --dry-run flag.

+
rclone move source:path dest:path [flags]
+

Options

+
      --delete-empty-src-dirs   Delete empty source dirs after move
+  -h, --help                    help for move
+

rclone delete

+

Remove the contents of path.

+

Synopsis

+

Remove the contents of path. Unlike purge it obeys include/exclude filters so can be used to selectively delete files.

+

Eg delete all files bigger than 100MBytes

+

Check what would be deleted first (use either)

+
rclone --min-size 100M lsl remote:path
+rclone --dry-run --min-size 100M delete remote:path
+

Then delete

+
rclone --min-size 100M delete remote:path
+

That reads "delete everything with a minimum size of 100 MB", hence delete all files bigger than 100MBytes.

+
rclone delete remote:path [flags]
+

Options

+
  -h, --help   help for delete
+

rclone purge

+

Remove the path and all of its contents.

+

Synopsis

+

Remove the path and all of its contents. Note that this does not obey include/exclude filters - everything will be removed. Use delete if you want to selectively delete files.

+
rclone purge remote:path [flags]
+

Options

+
  -h, --help   help for purge
+

rclone mkdir

+

Make the path if it doesn't already exist.

+

Synopsis

+

Make the path if it doesn't already exist.

+
rclone mkdir remote:path [flags]
+

Options

+
  -h, --help   help for mkdir
+

rclone rmdir

+

Remove the path if empty.

+

Synopsis

+

Remove the path. Note that you can't remove a path with objects in it, use purge for that.

+
rclone rmdir remote:path [flags]
+

Options

+
  -h, --help   help for rmdir
+

rclone check

+

Checks the files in the source and destination match.

+

Synopsis

+

Checks the files in the source and destination match. It compares sizes and hashes (MD5 or SHA1) and logs a report of files which don't match. It doesn't alter the source or destination.

+

If you supply the --size-only flag, it will only compare the sizes not the hashes as well. Use this for a quick check.

+

If you supply the --download flag, it will download the data from both remotes and check them against each other on the fly. This can be useful for remotes that don't support hashes or if you really want to check all the data.

+

If you supply the --one-way flag, it will only check that files in source match the files in destination, not the other way around. Meaning extra files in destination that are not in the source will not trigger an error.

+
rclone check source:path dest:path [flags]
+

Options

+
      --download   Check by downloading rather than with hash.
+  -h, --help       help for check
+      --one-way    Check one way only, source files must exist on remote
+

rclone ls

+

List the objects in the path with size and path.

+

Synopsis

+

Lists the objects in the source path to standard output in a human readable format with size and path. Recurses by default.

+

Eg

+
$ rclone ls swift:bucket
+    60295 bevajer5jef
+    90613 canole
+    94467 diwogej7
+    37600 fubuwic
+

Any of the filtering options can be applied to this commmand.

+

There are several related list commands

+ +

ls,lsl,lsd are designed to be human readable. lsf is designed to be human and machine readable. lsjson is designed to be machine readable.

+

Note that ls and lsl recurse by default - use "--max-depth 1" to stop the recursion.

+

The other list commands lsd,lsf,lsjson do not recurse by default - use "-R" to make them recurse.

+

Listing a non existent directory will produce an error except for remotes which can't have empty directories (eg s3, swift, gcs, etc - the bucket based remotes).

+
rclone ls remote:path [flags]
+

Options

+
  -h, --help   help for ls
+

rclone lsd

+

List all directories/containers/buckets in the path.

+

Synopsis

+

Lists the directories in the source path to standard output. Does not recurse by default. Use the -R flag to recurse.

+

This command lists the total size of the directory (if known, -1 if not), the modification time (if known, the current time if not), the number of objects in the directory (if known, -1 if not) and the name of the directory, Eg

+
$ rclone lsd swift:
+      494000 2018-04-26 08:43:20     10000 10000files
+          65 2018-04-26 08:43:20         1 1File
+

Or

+
$ rclone lsd drive:test
+          -1 2016-10-17 17:41:53        -1 1000files
+          -1 2017-01-03 14:40:54        -1 2500files
+          -1 2017-07-08 14:39:28        -1 4000files
+

If you just want the directory names use "rclone lsf --dirs-only".

+

Any of the filtering options can be applied to this commmand.

+

There are several related list commands

+ +

ls,lsl,lsd are designed to be human readable. lsf is designed to be human and machine readable. lsjson is designed to be machine readable.

+

Note that ls and lsl recurse by default - use "--max-depth 1" to stop the recursion.

+

The other list commands lsd,lsf,lsjson do not recurse by default - use "-R" to make them recurse.

+

Listing a non existent directory will produce an error except for remotes which can't have empty directories (eg s3, swift, gcs, etc - the bucket based remotes).

+
rclone lsd remote:path [flags]
+

Options

+
  -h, --help        help for lsd
+  -R, --recursive   Recurse into the listing.
+

rclone lsl

+

List the objects in path with modification time, size and path.

+

Synopsis

+

Lists the objects in the source path to standard output in a human readable format with modification time, size and path. Recurses by default.

+

Eg

+
$ rclone lsl swift:bucket
+    60295 2016-06-25 18:55:41.062626927 bevajer5jef
+    90613 2016-06-25 18:55:43.302607074 canole
+    94467 2016-06-25 18:55:43.046609333 diwogej7
+    37600 2016-06-25 18:55:40.814629136 fubuwic
+

Any of the filtering options can be applied to this commmand.

+

There are several related list commands

+ +

ls,lsl,lsd are designed to be human readable. lsf is designed to be human and machine readable. lsjson is designed to be machine readable.

+

Note that ls and lsl recurse by default - use "--max-depth 1" to stop the recursion.

+

The other list commands lsd,lsf,lsjson do not recurse by default - use "-R" to make them recurse.

+

Listing a non existent directory will produce an error except for remotes which can't have empty directories (eg s3, swift, gcs, etc - the bucket based remotes).

+
rclone lsl remote:path [flags]
+

Options

+
  -h, --help   help for lsl
+

rclone md5sum

+

Produces an md5sum file for all the objects in the path.

+

Synopsis

+

Produces an md5sum file for all the objects in the path. This is in the same format as the standard md5sum tool produces.

+
rclone md5sum remote:path [flags]
+

Options

+
  -h, --help   help for md5sum
+

rclone sha1sum

+

Produces an sha1sum file for all the objects in the path.

+

Synopsis

+

Produces an sha1sum file for all the objects in the path. This is in the same format as the standard sha1sum tool produces.

+
rclone sha1sum remote:path [flags]
+

Options

+
  -h, --help   help for sha1sum
+

rclone size

+

Prints the total size and number of objects in remote:path.

+

Synopsis

+

Prints the total size and number of objects in remote:path.

+
rclone size remote:path [flags]
+

Options

+
  -h, --help   help for size
+      --json   format output as JSON
+

rclone version

+

Show the version number.

+

Synopsis

+

Show the version number, the go version and the architecture.

+

Eg

+
$ rclone version
+rclone v1.41
+- os/arch: linux/amd64
+- go version: go1.10
+

If you supply the --check flag, then it will do an online check to compare your version with the latest release and the latest beta.

+
$ rclone version --check
+yours:  1.42.0.6
+latest: 1.42          (released 2018-06-16)
+beta:   1.42.0.5      (released 2018-06-17)
+

Or

+
$ rclone version --check
+yours:  1.41
+latest: 1.42          (released 2018-06-16)
+  upgrade: https://downloads.rclone.org/v1.42
+beta:   1.42.0.5      (released 2018-06-17)
+  upgrade: https://beta.rclone.org/v1.42-005-g56e1e820
+
rclone version [flags]
+

Options

+
      --check   Check for new version.
+  -h, --help    help for version
+

rclone cleanup

+

Clean up the remote if possible

+

Synopsis

+

Clean up the remote if possible. Empty the trash or delete old file versions. Not supported by all remotes.

+
rclone cleanup remote:path [flags]
+

Options

+
  -h, --help   help for cleanup
+

rclone dedupe

+

Interactively find duplicate files and delete/rename them.

+

Synopsis

+

By default dedupe interactively finds duplicate files and offers to delete all but one or rename them to be different. Only useful with Google Drive which can have duplicate file names.

+

In the first pass it will merge directories with the same name. It will do this iteratively until all the identical directories have been merged.

+

The dedupe command will delete all but one of any identical (same md5sum) files it finds without confirmation. This means that for most duplicated files the dedupe command will not be interactive. You can use --dry-run to see what would happen without doing anything.

+

Here is an example run.

+

Before - with duplicates

+
$ rclone lsl drive:dupes
+  6048320 2016-03-05 16:23:16.798000000 one.txt
+  6048320 2016-03-05 16:23:11.775000000 one.txt
+   564374 2016-03-05 16:23:06.731000000 one.txt
+  6048320 2016-03-05 16:18:26.092000000 one.txt
+  6048320 2016-03-05 16:22:46.185000000 two.txt
+  1744073 2016-03-05 16:22:38.104000000 two.txt
+   564374 2016-03-05 16:22:52.118000000 two.txt
+

Now the dedupe session

+
$ rclone dedupe drive:dupes
+2016/03/05 16:24:37 Google drive root 'dupes': Looking for duplicates using interactive mode.
+one.txt: Found 4 duplicates - deleting identical copies
+one.txt: Deleting 2/3 identical duplicates (md5sum "1eedaa9fe86fd4b8632e2ac549403b36")
+one.txt: 2 duplicates remain
+  1:      6048320 bytes, 2016-03-05 16:23:16.798000000, md5sum 1eedaa9fe86fd4b8632e2ac549403b36
+  2:       564374 bytes, 2016-03-05 16:23:06.731000000, md5sum 7594e7dc9fc28f727c42ee3e0749de81
+s) Skip and do nothing
+k) Keep just one (choose which in next step)
+r) Rename all to be different (by changing file.jpg to file-1.jpg)
+s/k/r> k
+Enter the number of the file to keep> 1
+one.txt: Deleted 1 extra copies
+two.txt: Found 3 duplicates - deleting identical copies
+two.txt: 3 duplicates remain
+  1:       564374 bytes, 2016-03-05 16:22:52.118000000, md5sum 7594e7dc9fc28f727c42ee3e0749de81
+  2:      6048320 bytes, 2016-03-05 16:22:46.185000000, md5sum 1eedaa9fe86fd4b8632e2ac549403b36
+  3:      1744073 bytes, 2016-03-05 16:22:38.104000000, md5sum 851957f7fb6f0bc4ce76be966d336802
+s) Skip and do nothing
+k) Keep just one (choose which in next step)
+r) Rename all to be different (by changing file.jpg to file-1.jpg)
+s/k/r> r
+two-1.txt: renamed from: two.txt
+two-2.txt: renamed from: two.txt
+two-3.txt: renamed from: two.txt
+

The result being

+
$ rclone lsl drive:dupes
+  6048320 2016-03-05 16:23:16.798000000 one.txt
+   564374 2016-03-05 16:22:52.118000000 two-1.txt
+  6048320 2016-03-05 16:22:46.185000000 two-2.txt
+  1744073 2016-03-05 16:22:38.104000000 two-3.txt
+

Dedupe can be run non interactively using the --dedupe-mode flag or by using an extra parameter with the same value

+ +

For example to rename all the identically named photos in your Google Photos directory, do

+
rclone dedupe --dedupe-mode rename "drive:Google Photos"
+

Or

+
rclone dedupe rename "drive:Google Photos"
+
rclone dedupe [mode] remote:path [flags]
+

Options

+
      --dedupe-mode string   Dedupe mode interactive|skip|first|newest|oldest|rename. (default "interactive")
+  -h, --help                 help for dedupe
+

rclone about

+

Get quota information from the remote.

+

Synopsis

+

Get quota information from the remote, like bytes used/free/quota and bytes used in the trash. Not supported by all remotes.

+

This will print to stdout something like this:

+
Total:   17G
+Used:    7.444G
+Free:    1.315G
+Trashed: 100.000M
+Other:   8.241G
+

Where the fields are:

+ +

Note that not all the backends provide all the fields - they will be missing if they are not known for that backend. Where it is known that the value is unlimited the value will also be omitted.

+

Use the --full flag to see the numbers written out in full, eg

+
Total:   18253611008
+Used:    7993453766
+Free:    1411001220
+Trashed: 104857602
+Other:   8849156022
+

Use the --json flag for a computer readable output, eg

+
{
+    "total": 18253611008,
+    "used": 7993453766,
+    "trashed": 104857602,
+    "other": 8849156022,
+    "free": 1411001220
+}
+
rclone about remote: [flags]
+

Options

+
      --full   Full numbers instead of SI units
+  -h, --help   help for about
+      --json   Format output as JSON
+

rclone authorize

+

Remote authorization.

+

Synopsis

+

Remote authorization. Used to authorize a remote or headless rclone from a machine with a browser - use as instructed by rclone config.

+
rclone authorize [flags]
+

Options

+
  -h, --help   help for authorize
+

rclone cachestats

+

Print cache stats for a remote

+

Synopsis

+

Print cache stats for a remote in JSON format

+
rclone cachestats source: [flags]
+

Options

+
  -h, --help   help for cachestats
+

rclone cat

+

Concatenates any files and sends them to stdout.

+

Synopsis

+

rclone cat sends any files to standard output.

+

You can use it like this to output a single file

+
rclone cat remote:path/to/file
+

Or like this to output any file in dir or subdirectories.

+
rclone cat remote:path/to/dir
+

Or like this to output any .txt files in dir or subdirectories.

+
rclone --include "*.txt" cat remote:path/to/dir
+

Use the --head flag to print characters only at the start, --tail for the end and --offset and --count to print a section in the middle. Note that if offset is negative it will count from the end, so --offset -1 --count 1 is equivalent to --tail 1.

+
rclone cat remote:path [flags]
+

Options

+
      --count int    Only print N characters. (default -1)
+      --discard      Discard the output instead of printing.
+      --head int     Only print the first N characters.
+  -h, --help         help for cat
+      --offset int   Start printing at offset N (or from end if -ve).
+      --tail int     Only print the last N characters.
+

rclone config create

+

Create a new remote with name, type and options.

+

Synopsis

+

Create a new remote of with and options. The options should be passed in in pairs of .

+

For example to make a swift remote of name myremote using auto config you would do:

+
rclone config create myremote swift env_auth true
+
rclone config create <name> <type> [<key> <value>]* [flags]
+

Options

+
  -h, --help   help for create
+

rclone config delete

+

Delete an existing remote .

+

Synopsis

+

Delete an existing remote .

+
rclone config delete <name> [flags]
+

Options

+
  -h, --help   help for delete
+

rclone config dump

+

Dump the config file as JSON.

+

Synopsis

+

Dump the config file as JSON.

+
rclone config dump [flags]
+

Options

+
  -h, --help   help for dump
+

rclone config edit

+

Enter an interactive configuration session.

+

Synopsis

+

Enter an interactive configuration session where you can setup new remotes and manage existing ones. You may also set or remove a password to protect your configuration.

+
rclone config edit [flags]
+

Options

+
  -h, --help   help for edit
+

rclone config file

+

Show path of configuration file in use.

+

Synopsis

+

Show path of configuration file in use.

+
rclone config file [flags]
+

Options

+
  -h, --help   help for file
+

rclone config password

+

Update password in an existing remote.

+

Synopsis

+

Update an existing remote's password. The password should be passed in in pairs of .

+

For example to set password of a remote of name myremote you would do:

+
rclone config password myremote fieldname mypassword
+
rclone config password <name> [<key> <value>]+ [flags]
+

Options

+
  -h, --help   help for password
+

rclone config providers

+

List in JSON format all the providers and options.

+

Synopsis

+

List in JSON format all the providers and options.

+
rclone config providers [flags]
+

Options

+
  -h, --help   help for providers
+

rclone config show

+

Print (decrypted) config file, or the config for a single remote.

+

Synopsis

+

Print (decrypted) config file, or the config for a single remote.

+
rclone config show [<remote>] [flags]
+

Options

+
  -h, --help   help for show
+

rclone config update

+

Update options in an existing remote.

+

Synopsis

+

Update an existing remote's options. The options should be passed in in pairs of .

+

For example to update the env_auth field of a remote of name myremote you would do:

+
rclone config update myremote swift env_auth true
+
rclone config update <name> [<key> <value>]+ [flags]
+

Options

+
  -h, --help   help for update
+

rclone copyto

+

Copy files from source to dest, skipping already copied

+

Synopsis

+

If source:path is a file or directory then it copies it to a file or directory named dest:path.

+

This can be used to upload single files to other than their current name. If the source is a directory then it acts exactly like the copy command.

+

So

+
rclone copyto src dst
+

where src and dst are rclone paths, either remote:path or /path/to/local or C:.

+

This will:

+
if src is file
+    copy it to dst, overwriting an existing file if it exists
+if src is directory
+    copy it to dst, overwriting existing files if they exist
+    see copy command for full details
+

This doesn't transfer unchanged files, testing by size and modification time or MD5SUM. It doesn't delete files from the destination.

+
rclone copyto source:path dest:path [flags]
+

Options

+
  -h, --help   help for copyto
+

rclone copyurl

+

Copy url content to dest.

+

Synopsis

+

Download urls content and copy it to destination without saving it in tmp storage.

+
rclone copyurl https://example.com dest:path [flags]
+

Options

+
  -h, --help   help for copyurl
+

rclone cryptcheck

+

Cryptcheck checks the integrity of a crypted remote.

+

Synopsis

+

rclone cryptcheck checks a remote against a crypted remote. This is the equivalent of running rclone check, but able to check the checksums of the crypted remote.

+

For it to work the underlying remote of the cryptedremote must support some kind of checksum.

+

It works by reading the nonce from each file on the cryptedremote: and using that to encrypt each file on the remote:. It then checks the checksum of the underlying file on the cryptedremote: against the checksum of the file it has just encrypted.

+

Use it like this

+
rclone cryptcheck /path/to/files encryptedremote:path
+

You can use it like this also, but that will involve downloading all the files in remote:path.

+
rclone cryptcheck remote:path encryptedremote:path
+

After it has run it will log the status of the encryptedremote:.

+

If you supply the --one-way flag, it will only check that files in source match the files in destination, not the other way around. Meaning extra files in destination that are not in the source will not trigger an error.

+
rclone cryptcheck remote:path cryptedremote:path [flags]
+

Options

+
  -h, --help      help for cryptcheck
+      --one-way   Check one way only, source files must exist on destination
+

rclone cryptdecode

+

Cryptdecode returns unencrypted file names.

+

Synopsis

+

rclone cryptdecode returns unencrypted file names when provided with a list of encrypted file names. List limit is 10 items.

+

If you supply the --reverse flag, it will return encrypted file names.

+

use it like this

+
rclone cryptdecode encryptedremote: encryptedfilename1 encryptedfilename2
+
+rclone cryptdecode --reverse encryptedremote: filename1 filename2
+
rclone cryptdecode encryptedremote: encryptedfilename [flags]
+

Options

+
  -h, --help      help for cryptdecode
+      --reverse   Reverse cryptdecode, encrypts filenames
+

rclone dbhashsum

+

Produces a Dropbox hash file for all the objects in the path.

+

Synopsis

+

Produces a Dropbox hash file for all the objects in the path. The hashes are calculated according to Dropbox content hash rules. The output is in the same format as md5sum and sha1sum.

+
rclone dbhashsum remote:path [flags]
+

Options

+
  -h, --help   help for dbhashsum
+

rclone deletefile

+

Remove a single file from remote.

+

Synopsis

+

Remove a single file from remote. Unlike delete it cannot be used to remove a directory and it doesn't obey include/exclude filters - if the specified file exists, it will always be removed.

+
rclone deletefile remote:path [flags]
+

Options

+
  -h, --help   help for deletefile
+

rclone genautocomplete

+

Output completion script for a given shell.

+

Synopsis

+

Generates a shell completion script for rclone. Run with --help to list the supported shells.

+

Options

+
  -h, --help   help for genautocomplete
+

rclone genautocomplete bash

+

Output bash completion script for rclone.

+

Synopsis

+

Generates a bash shell autocompletion script for rclone.

+

This writes to /etc/bash_completion.d/rclone by default so will probably need to be run with sudo or as root, eg

+
sudo rclone genautocomplete bash
+

Logout and login again to use the autocompletion scripts, or source them directly

+
. /etc/bash_completion
+

If you supply a command line argument the script will be written there.

+
rclone genautocomplete bash [output_file] [flags]
+

Options

+
  -h, --help   help for bash
+

rclone genautocomplete zsh

+

Output zsh completion script for rclone.

+

Synopsis

+

Generates a zsh autocompletion script for rclone.

+

This writes to /usr/share/zsh/vendor-completions/_rclone by default so will probably need to be run with sudo or as root, eg

+
sudo rclone genautocomplete zsh
+

Logout and login again to use the autocompletion scripts, or source them directly

+
autoload -U compinit && compinit
+

If you supply a command line argument the script will be written there.

+
rclone genautocomplete zsh [output_file] [flags]
+

Options

+
  -h, --help   help for zsh
+

rclone gendocs

+

Output markdown docs for rclone to the directory supplied.

+

Synopsis

+

This produces markdown docs for the rclone commands to the directory supplied. These are in a format suitable for hugo to render into the rclone.org website.

+
rclone gendocs output_directory [flags]
+

Options

+
  -h, --help   help for gendocs
+

rclone hashsum

+

Produces an hashsum file for all the objects in the path.

+

Synopsis

+

Produces a hash file for all the objects in the path using the hash named. The output is in the same format as the standard md5sum/sha1sum tool.

+

Run without a hash to see the list of supported hashes, eg

+
$ rclone hashsum
+Supported hashes are:
+  * MD5
+  * SHA-1
+  * DropboxHash
+  * QuickXorHash
+

Then

+
$ rclone hashsum MD5 remote:path
+
rclone hashsum <hash> remote:path [flags]
+

Options

+
  -h, --help   help for hashsum
+ +

Generate public link to file/folder.

+

Synopsis

+

rclone link will create or retrieve a public link to the given file or folder.

+
rclone link remote:path/to/file
+rclone link remote:path/to/folder/
+

If successful, the last line of the output will contain the link. Exact capabilities depend on the remote, but the link will always be created with the least constraints – e.g. no expiry, no password protection, accessible without account.

+
rclone link remote:path [flags]
+

Options

+
  -h, --help   help for link
+

rclone listremotes

+

List all the remotes in the config file.

+

Synopsis

+

rclone listremotes lists all the available remotes from the config file.

+

When uses with the -l flag it lists the types too.

+
rclone listremotes [flags]
+

Options

+
  -h, --help   help for listremotes
+  -l, --long   Show the type as well as names.
+

rclone lsf

+

List directories and objects in remote:path formatted for parsing

+

Synopsis

+

List the contents of the source path (directories and objects) to standard output in a form which is easy to parse by scripts. By default this will just be the names of the objects and directories, one per line. The directories will have a / suffix.

+

Eg

+
$ rclone lsf swift:bucket
+bevajer5jef
+canole
+diwogej7
+ferejej3gux/
+fubuwic
+

Use the --format option to control what gets listed. By default this is just the path, but you can use these parameters to control the output:

+
p - path
+s - size
+t - modification time
+h - hash
+i - ID of object if known
+m - MimeType of object if known
+

So if you wanted the path, size and modification time, you would use --format "pst", or maybe --format "tsp" to put the path last.

+

Eg

+
$ rclone lsf  --format "tsp" swift:bucket
+2016-06-25 18:55:41;60295;bevajer5jef
+2016-06-25 18:55:43;90613;canole
+2016-06-25 18:55:43;94467;diwogej7
+2018-04-26 08:50:45;0;ferejej3gux/
+2016-06-25 18:55:40;37600;fubuwic
+

If you specify "h" in the format you will get the MD5 hash by default, use the "--hash" flag to change which hash you want. Note that this can be returned as an empty string if it isn't available on the object (and for directories), "ERROR" if there was an error reading it from the object and "UNSUPPORTED" if that object does not support that hash type.

+

For example to emulate the md5sum command you can use

+
rclone lsf -R --hash MD5 --format hp --separator "  " --files-only .
+

Eg

+
$ rclone lsf -R --hash MD5 --format hp --separator "  " --files-only swift:bucket 
+7908e352297f0f530b84a756f188baa3  bevajer5jef
+cd65ac234e6fea5925974a51cdd865cc  canole
+03b5341b4f234b9d984d03ad076bae91  diwogej7
+8fd37c3810dd660778137ac3a66cc06d  fubuwic
+99713e14a4c4ff553acaf1930fad985b  gixacuh7ku
+

(Though "rclone md5sum ." is an easier way of typing this.)

+

By default the separator is ";" this can be changed with the --separator flag. Note that separators aren't escaped in the path so putting it last is a good strategy.

+

Eg

+
$ rclone lsf  --separator "," --format "tshp" swift:bucket
+2016-06-25 18:55:41,60295,7908e352297f0f530b84a756f188baa3,bevajer5jef
+2016-06-25 18:55:43,90613,cd65ac234e6fea5925974a51cdd865cc,canole
+2016-06-25 18:55:43,94467,03b5341b4f234b9d984d03ad076bae91,diwogej7
+2018-04-26 08:52:53,0,,ferejej3gux/
+2016-06-25 18:55:40,37600,8fd37c3810dd660778137ac3a66cc06d,fubuwic
+

You can output in CSV standard format. This will escape things in " if they contain ,

+

Eg

+
$ rclone lsf --csv --files-only --format ps remote:path
+test.log,22355
+test.sh,449
+"this file contains a comma, in the file name.txt",6
+

Note that the --absolute parameter is useful for making lists of files to pass to an rclone copy with the --files-from flag.

+

For example to find all the files modified within one day and copy those only (without traversing the whole directory structure):

+
rclone lsf --absolute --files-only --max-age 1d /path/to/local > new_files
+rclone copy --files-from new_files /path/to/local remote:path
+

Any of the filtering options can be applied to this commmand.

+

There are several related list commands

+ +

ls,lsl,lsd are designed to be human readable. lsf is designed to be human and machine readable. lsjson is designed to be machine readable.

+

Note that ls and lsl recurse by default - use "--max-depth 1" to stop the recursion.

+

The other list commands lsd,lsf,lsjson do not recurse by default - use "-R" to make them recurse.

+

Listing a non existent directory will produce an error except for remotes which can't have empty directories (eg s3, swift, gcs, etc - the bucket based remotes).

+
rclone lsf remote:path [flags]
+

Options

+
      --absolute           Put a leading / in front of path names.
+      --csv                Output in CSV format.
+  -d, --dir-slash          Append a slash to directory names. (default true)
+      --dirs-only          Only list directories.
+      --files-only         Only list files.
+  -F, --format string      Output format - see  help for details (default "p")
+      --hash h             Use this hash when h is used in the format MD5|SHA-1|DropboxHash (default "MD5")
+  -h, --help               help for lsf
+  -R, --recursive          Recurse into the listing.
+  -s, --separator string   Separator for the items in the format. (default ";")
+

rclone lsjson

+

List directories and objects in the path in JSON format.

+

Synopsis

+

List directories and objects in the path in JSON format.

+

The output is an array of Items, where each Item looks like this

+

{ "Hashes" : { "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f", "MD5" : "b1946ac92492d2347c6235b4d2611184", "DropboxHash" : "ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc" }, "ID": "y2djkhiujf83u33", "OrigID": "UYOJVTUW00Q1RzTDA", "IsDir" : false, "MimeType" : "application/octet-stream", "ModTime" : "2017-05-31T16:15:57.034468261+01:00", "Name" : "file.txt", "Encrypted" : "v0qpsdq8anpci8n929v3uu9338", "Path" : "full/path/goes/here/file.txt", "Size" : 6 }

+

If --hash is not specified the Hashes property won't be emitted.

+

If --no-modtime is specified then ModTime will be blank.

+

If --encrypted is not specified the Encrypted won't be emitted.

+

The Path field will only show folders below the remote path being listed. If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt" will be "subfolder/file.txt", not "remote:path/subfolder/file.txt". When used without --recursive the Path will always be the same as Name.

+

The time is in RFC3339 format with nanosecond precision.

+

The whole output can be processed as a JSON blob, or alternatively it can be processed line by line as each item is written one to a line.

+

Any of the filtering options can be applied to this commmand.

+

There are several related list commands

+ +

ls,lsl,lsd are designed to be human readable. lsf is designed to be human and machine readable. lsjson is designed to be machine readable.

+

Note that ls and lsl recurse by default - use "--max-depth 1" to stop the recursion.

+

The other list commands lsd,lsf,lsjson do not recurse by default - use "-R" to make them recurse.

+

Listing a non existent directory will produce an error except for remotes which can't have empty directories (eg s3, swift, gcs, etc - the bucket based remotes).

+
rclone lsjson remote:path [flags]
+

Options

+
  -M, --encrypted    Show the encrypted names.
+      --hash         Include hashes in the output (may take longer).
+  -h, --help         help for lsjson
+      --no-modtime   Don't read the modification time (can speed things up).
+      --original     Show the ID of the underlying Object.
+  -R, --recursive    Recurse into the listing.
+

rclone mount

+

Mount the remote as a mountpoint. EXPERIMENTAL

+

Synopsis

+

rclone mount allows Linux, FreeBSD, macOS and Windows to mount any of Rclone's cloud storage systems as a file system with FUSE.

+

This is EXPERIMENTAL - use with care.

+

First set up your remote using rclone config. Check it works with rclone ls etc.

+

Start the mount like this

+
rclone mount remote:path/to/files /path/to/local/mount
+

Or on Windows like this where X: is an unused drive letter

+
rclone mount remote:path/to/files X:
+

When the program ends, either via Ctrl+C or receiving a SIGINT or SIGTERM signal, the mount is automatically stopped.

+

The umount operation can fail, for example when the mountpoint is busy. When that happens, it is the user's responsibility to stop the mount manually with

+
# Linux
+fusermount -u /path/to/local/mount
+# OS X
+umount /path/to/local/mount
+

Installing on Windows

+

To run rclone mount on Windows, you will need to download and install WinFsp.

+

WinFsp is an open source Windows File System Proxy which makes it easy to write user space file systems for Windows. It provides a FUSE emulation layer which rclone uses combination with cgofuse. Both of these packages are by Bill Zissimopoulos who was very helpful during the implementation of rclone mount for Windows.

+

Windows caveats

+

Note that drives created as Administrator are not visible by other accounts (including the account that was elevated as Administrator). So if you start a Windows drive from an Administrative Command Prompt and then try to access the same drive from Explorer (which does not run as Administrator), you will not be able to see the new drive.

+

The easiest way around this is to start the drive from a normal command prompt. It is also possible to start a drive from the SYSTEM account (using the WinFsp.Launcher infrastructure) which creates drives accessible for everyone on the system or alternatively using the nssm service manager.

+

Limitations

+

Without the use of "--vfs-cache-mode" this can only write files sequentially, it can only seek when reading. This means that many applications won't work with their files on an rclone mount without "--vfs-cache-mode writes" or "--vfs-cache-mode full". See the File Caching section for more info.

+

The bucket based remotes (eg Swift, S3, Google Compute Storage, B2, Hubic) won't work from the root - you will need to specify a bucket, or a path within the bucket. So swift: won't work whereas swift:bucket will as will swift:bucket/path. None of these support the concept of directories, so empty directories will have a tendency to disappear once they fall out of the directory cache.

+

Only supported on Linux, FreeBSD, OS X and Windows at the moment.

+

rclone mount vs rclone sync/copy

+

File systems expect things to be 100% reliable, whereas cloud storage systems are a long way from 100% reliable. The rclone sync/copy commands cope with this with lots of retries. However rclone mount can't use retries in the same way without making local copies of the uploads. Look at the EXPERIMENTAL file caching for solutions to make mount mount more reliable.

+

Attribute caching

+

You can use the flag --attr-timeout to set the time the kernel caches the attributes (size, modification time etc) for directory entries.

+

The default is "1s" which caches files just long enough to avoid too many callbacks to rclone from the kernel.

+

In theory 0s should be the correct value for filesystems which can change outside the control of the kernel. However this causes quite a few problems such as rclone using too much memory, rclone not serving files to samba and excessive time listing directories.

+

The kernel can cache the info about a file for the time given by "--attr-timeout". You may see corruption if the remote file changes length during this window. It will show up as either a truncated file or a file with garbage on the end. With "--attr-timeout 1s" this is very unlikely but not impossible. The higher you set "--attr-timeout" the more likely it is. The default setting of "1s" is the lowest setting which mitigates the problems above.

+

If you set it higher ('10s' or '1m' say) then the kernel will call back to rclone less often making it more efficient, however there is more chance of the corruption issue above.

+

If files don't change on the remote outside of the control of rclone then there is no chance of corruption.

+

This is the same as setting the attr_timeout option in mount.fuse.

+

Filters

+

Note that all the rclone filters can be used to select a subset of the files to be visible in the mount.

+

systemd

+

When running rclone mount as a systemd service, it is possible to use Type=notify. In this case the service will enter the started state after the mountpoint has been successfully set up. Units having the rclone mount service specified as a requirement will see all files and folders immediately in this mode.

+

chunked reading

+

--vfs-read-chunk-size will enable reading the source objects in parts. This can reduce the used download quota for some remotes by requesting only chunks from the remote that are actually read at the cost of an increased number of requests.

+

When --vfs-read-chunk-size-limit is also specified and greater than --vfs-read-chunk-size, the chunk size for each open file will get doubled for each chunk read, until the specified value is reached. A value of -1 will disable the limit and the chunk size will grow indefinitely.

+

With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the following parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on. When --vfs-read-chunk-size-limit 500M is specified, the result would be 0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on.

+

Chunked reading will only work with --vfs-cache-mode < full, as the file will always be copied to the vfs cache before opening with --vfs-cache-mode full.

+

Directory Cache

+

Using the --dir-cache-time flag, you can set how long a directory should be considered up to date and not refreshed from the backend. Changes made locally in the mount may appear immediately or invalidate the cache. However, changes done on the remote will only be picked up once the cache expires.

+

Alternatively, you can send a SIGHUP signal to rclone for it to flush all directory caches, regardless of how old they are. Assuming only one rclone instance is running, you can reset the cache like this:

+
kill -SIGHUP $(pidof rclone)
+

If you configure rclone with a remote control then you can use rclone rc to flush the whole directory cache:

+
rclone rc vfs/forget
+

Or individual files or directories:

+
rclone rc vfs/forget file=path/to/file dir=path/to/dir
+

File Buffering

+

The --buffer-size flag determines the amount of memory, that will be used to buffer data in advance.

+

Each open file descriptor will try to keep the specified amount of data in memory at all times. The buffered data is bound to one file descriptor and won't be shared between multiple open file descriptors of the same file.

+

This flag is a upper limit for the used memory per file descriptor. The buffer will only use memory for data that is downloaded but not not yet read. If the buffer is empty, only a small amount of memory will be used. The maximum memory used by rclone for buffering can be up to --buffer-size * open files.

+

File Caching

+

NB File caching is EXPERIMENTAL - use with care!

+

These flags control the VFS file caching options. The VFS layer is used by rclone mount to make a cloud storage system work more like a normal file system.

+

You'll need to enable VFS caching if you want, for example, to read and write simultaneously to a file. See below for more details.

+

Note that the VFS cache works in addition to the cache backend and you may find that you need one or the other or both.

+
--cache-dir string                   Directory rclone will use for caching.
+--vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
+--vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
+--vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+

If run with -vv rclone will print the location of the file cache. The files are stored in the user cache file area which is OS dependent but can be controlled with --cache-dir or setting the appropriate environment variable.

+

The cache has 4 different modes selected by --vfs-cache-mode. The higher the cache mode the more compatible rclone becomes at the cost of using disk space.

+

Note that files are written back to the remote only when they are closed so if rclone is quit or dies with open files then these won't get written back to the remote. However they will still be in the on disk cache.

+

--vfs-cache-mode off

+

In this mode the cache will read directly from the remote and write directly to the remote without caching anything on disk.

+

This will mean some operations are not possible

+ +

--vfs-cache-mode minimal

+

This is very similar to "off" except that files opened for read AND write will be buffered to disks. This means that files opened for write will be a lot more compatible, but uses the minimal disk space.

+

These operations are not possible

+ +

--vfs-cache-mode writes

+

In this mode files opened for read only are still read directly from the remote, write only and read/write files are buffered to disk first.

+

This mode should support all normal file system operations.

+

If an upload fails it will be retried up to --low-level-retries times.

+

--vfs-cache-mode full

+

In this mode all reads and writes are buffered to and from disk. When a file is opened for read it will be downloaded in its entirety first.

+

This may be appropriate for your needs, or you may prefer to look at the cache backend which does a much more sophisticated job of caching, including caching directory hierarchies and chunks of files.

+

In this mode, unlike the others, when a file is written to the disk, it will be kept on the disk after it is written to the remote. It will be purged on a schedule according to --vfs-cache-max-age.

+

This mode should support all normal file system operations.

+

If an upload or download fails it will be retried up to --low-level-retries times.

+
rclone mount remote:path /path/to/mountpoint [flags]
+

Options

+
      --allow-non-empty                    Allow mounting over a non-empty directory.
+      --allow-other                        Allow access to other users.
+      --allow-root                         Allow access to root user.
+      --attr-timeout duration              Time for which file/directory attributes are cached. (default 1s)
+      --daemon                             Run mount as a daemon (background mode).
+      --daemon-timeout duration            Time limit for rclone to respond to kernel (not supported by all OSes).
+      --debug-fuse                         Debug the FUSE internals - needs -v.
+      --default-permissions                Makes kernel enforce access control based on the file mode.
+      --dir-cache-time duration            Time to cache directory entries for. (default 5m0s)
+      --fuse-flag stringArray              Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required.
+      --gid uint32                         Override the gid field set by the filesystem. (default 502)
+  -h, --help                               help for mount
+      --max-read-ahead int                 The number of bytes that can be prefetched for sequential reads. (default 128k)
+      --no-checksum                        Don't compare checksums on up/download.
+      --no-modtime                         Don't read/write the modification time (can speed things up).
+      --no-seek                            Don't allow seeking in files.
+  -o, --option stringArray                 Option for libfuse/WinFsp. Repeat if required.
+      --poll-interval duration             Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable. (default 1m0s)
+      --read-only                          Mount read-only.
+      --uid uint32                         Override the uid field set by the filesystem. (default 502)
+      --umask int                          Override the permission bits set by the filesystem.
+      --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
+      --vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
+      --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+      --vfs-read-chunk-size int            Read the source objects in chunks. (default 128M)
+      --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off)
+      --volname string                     Set the volume name (not supported by all OSes).
+      --write-back-cache                   Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.
+

rclone moveto

+

Move file or directory from source to dest.

+

Synopsis

+

If source:path is a file or directory then it moves it to a file or directory named dest:path.

+

This can be used to rename files or upload single files to other than their existing name. If the source is a directory then it acts exacty like the move command.

+

So

+
rclone moveto src dst
+

where src and dst are rclone paths, either remote:path or /path/to/local or C:.

+

This will:

+
if src is file
+    move it to dst, overwriting an existing file if it exists
+if src is directory
+    move it to dst, overwriting existing files if they exist
+    see move command for full details
+

This doesn't transfer unchanged files, testing by size and modification time or MD5SUM. src will be deleted on successful transfer.

+

Important: Since this can cause data loss, test first with the --dry-run flag.

+
rclone moveto source:path dest:path [flags]
+

Options

+
  -h, --help   help for moveto
+

rclone ncdu

+

Explore a remote with a text based user interface.

+

Synopsis

+

This displays a text based user interface allowing the navigation of a remote. It is most useful for answering the question - "What is using all my disk space?".

+ +

To make the user interface it first scans the entire remote given and builds an in memory representation. rclone ncdu can be used during this scanning phase and you will see it building up the directory structure as it goes along.

+

Here are the keys - press '?' to toggle the help on and off

+
 ↑,↓ or k,j to Move
+ →,l to enter
+ ←,h to return
+ c toggle counts
+ g toggle graph
+ n,s,C sort by name,size,count
+ ^L refresh screen
+ ? to toggle help on and off
+ q/ESC/c-C to quit
+

This an homage to the ncdu tool but for rclone remotes. It is missing lots of features at the moment, most importantly deleting files, but is useful as it stands.

+
rclone ncdu remote:path [flags]
+

Options

+
  -h, --help   help for ncdu
+

rclone obscure

+

Obscure password for use in the rclone.conf

+

Synopsis

+

Obscure password for use in the rclone.conf

+
rclone obscure password [flags]
+

Options

+
  -h, --help   help for obscure
+

rclone rc

+

Run a command against a running rclone.

+

Synopsis

+

This runs a command against a running rclone. By default it will use that specified in the --rc-addr command.

+

Arguments should be passed in as parameter=value.

+

The result will be returned as a JSON object by default.

+

Use "rclone rc" to see a list of all possible commands.

+
rclone rc commands parameter [flags]
+

Options

+
  -h, --help         help for rc
+      --no-output    If set don't output the JSON result.
+      --url string   URL to connect to rclone remote control. (default "http://localhost:5572/")
+

rclone rcat

+

Copies standard input to file on remote.

+

Synopsis

+

rclone rcat reads from standard input (stdin) and copies it to a single remote file.

+
echo "hello world" | rclone rcat remote:path/to/file
+ffmpeg - | rclone rcat remote:path/to/file
+

If the remote file already exists, it will be overwritten.

+

rcat will try to upload small files in a single request, which is usually more efficient than the streaming/chunked upload endpoints, which use multiple requests. Exact behaviour depends on the remote. What is considered a small file may be set through --streaming-upload-cutoff. Uploading only starts after the cutoff is reached or if the file ends before that. The data must fit into RAM. The cutoff needs to be small enough to adhere the limits of your remote, please see there. Generally speaking, setting this cutoff too high will decrease your performance.

+

Note that the upload can also not be retried because the data is not kept around until the upload succeeds. If you need to transfer a lot of data, you're better off caching locally and then rclone move it to the destination.

+
rclone rcat remote:path [flags]
+

Options

+
  -h, --help   help for rcat
+

rclone rmdirs

+

Remove empty directories under the path.

+

Synopsis

+

This removes any empty directories (or directories that only contain empty directories) under the path that it finds, including the path if it has nothing in.

+

If you supply the --leave-root flag, it will not remove the root directory.

+

This is useful for tidying up remotes that rclone has left a lot of empty directories in.

+
rclone rmdirs remote:path [flags]
+

Options

+
  -h, --help         help for rmdirs
+      --leave-root   Do not remove root directory if empty
+

rclone serve

+

Serve a remote over a protocol.

+

Synopsis

+

rclone serve is used to serve a remote over a given protocol. This command requires the use of a subcommand to specify the protocol, eg

+
rclone serve http remote:
+

Each subcommand has its own options which you can see in their help.

+
rclone serve <protocol> [opts] <remote> [flags]
+

Options

+
  -h, --help   help for serve
+

rclone serve http

+

Serve the remote over HTTP.

+

Synopsis

+

rclone serve http implements a basic web server to serve the remote over HTTP. This can be viewed in a web browser or you can make a remote of type http read from it.

+

You can use the filter flags (eg --include, --exclude) to control what is served.

+

The server will log errors. Use -v to see access logs.

+

--bwlimit will be respected for file transfers. Use --stats to control the stats printing.

+

Server options

+

Use --addr to specify which IP address and port the server should listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.

+

If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.

+

--server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.

+

--max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.

+

Authentication

+

By default this will serve files without needing a login.

+

You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.

+

Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.

+

To create an htpasswd file:

+
touch htpasswd
+htpasswd -B htpasswd user
+htpasswd -B htpasswd anotherUser
+

The password file can be updated while rclone is running.

+

Use --realm to set the authentication realm.

+

SSL/TLS

+

By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.

+

--cert should be a either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.

+

Directory Cache

+

Using the --dir-cache-time flag, you can set how long a directory should be considered up to date and not refreshed from the backend. Changes made locally in the mount may appear immediately or invalidate the cache. However, changes done on the remote will only be picked up once the cache expires.

+

Alternatively, you can send a SIGHUP signal to rclone for it to flush all directory caches, regardless of how old they are. Assuming only one rclone instance is running, you can reset the cache like this:

+
kill -SIGHUP $(pidof rclone)
+

If you configure rclone with a remote control then you can use rclone rc to flush the whole directory cache:

+
rclone rc vfs/forget
+

Or individual files or directories:

+
rclone rc vfs/forget file=path/to/file dir=path/to/dir
+

File Buffering

+

The --buffer-size flag determines the amount of memory, that will be used to buffer data in advance.

+

Each open file descriptor will try to keep the specified amount of data in memory at all times. The buffered data is bound to one file descriptor and won't be shared between multiple open file descriptors of the same file.

+

This flag is a upper limit for the used memory per file descriptor. The buffer will only use memory for data that is downloaded but not not yet read. If the buffer is empty, only a small amount of memory will be used. The maximum memory used by rclone for buffering can be up to --buffer-size * open files.

+

File Caching

+

NB File caching is EXPERIMENTAL - use with care!

+

These flags control the VFS file caching options. The VFS layer is used by rclone mount to make a cloud storage system work more like a normal file system.

+

You'll need to enable VFS caching if you want, for example, to read and write simultaneously to a file. See below for more details.

+

Note that the VFS cache works in addition to the cache backend and you may find that you need one or the other or both.

+
--cache-dir string                   Directory rclone will use for caching.
+--vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
+--vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
+--vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+

If run with -vv rclone will print the location of the file cache. The files are stored in the user cache file area which is OS dependent but can be controlled with --cache-dir or setting the appropriate environment variable.

+

The cache has 4 different modes selected by --vfs-cache-mode. The higher the cache mode the more compatible rclone becomes at the cost of using disk space.

+

Note that files are written back to the remote only when they are closed so if rclone is quit or dies with open files then these won't get written back to the remote. However they will still be in the on disk cache.

+

--vfs-cache-mode off

+

In this mode the cache will read directly from the remote and write directly to the remote without caching anything on disk.

+

This will mean some operations are not possible

+ +

--vfs-cache-mode minimal

+

This is very similar to "off" except that files opened for read AND write will be buffered to disks. This means that files opened for write will be a lot more compatible, but uses the minimal disk space.

+

These operations are not possible

+ +

--vfs-cache-mode writes

+

In this mode files opened for read only are still read directly from the remote, write only and read/write files are buffered to disk first.

+

This mode should support all normal file system operations.

+

If an upload fails it will be retried up to --low-level-retries times.

+

--vfs-cache-mode full

+

In this mode all reads and writes are buffered to and from disk. When a file is opened for read it will be downloaded in its entirety first.

+

This may be appropriate for your needs, or you may prefer to look at the cache backend which does a much more sophisticated job of caching, including caching directory hierarchies and chunks of files.

+

In this mode, unlike the others, when a file is written to the disk, it will be kept on the disk after it is written to the remote. It will be purged on a schedule according to --vfs-cache-max-age.

+

This mode should support all normal file system operations.

+

If an upload or download fails it will be retried up to --low-level-retries times.

+
rclone serve http remote:path [flags]
+

Options

+
      --addr string                        IPaddress:Port or :Port to bind server to. (default "localhost:8080")
+      --cert string                        SSL PEM key (concatenation of certificate and CA certificate)
+      --client-ca string                   Client certificate authority to verify clients with
+      --dir-cache-time duration            Time to cache directory entries for. (default 5m0s)
+      --gid uint32                         Override the gid field set by the filesystem. (default 502)
+  -h, --help                               help for http
+      --htpasswd string                    htpasswd file - if not provided no authentication is done
+      --key string                         SSL PEM Private key
+      --max-header-bytes int               Maximum size of request header (default 4096)
+      --no-checksum                        Don't compare checksums on up/download.
+      --no-modtime                         Don't read/write the modification time (can speed things up).
+      --no-seek                            Don't allow seeking in files.
+      --pass string                        Password for authentication.
+      --poll-interval duration             Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable. (default 1m0s)
+      --read-only                          Mount read-only.
+      --realm string                       realm for authentication (default "rclone")
+      --server-read-timeout duration       Timeout for server reading data (default 1h0m0s)
+      --server-write-timeout duration      Timeout for server writing data (default 1h0m0s)
+      --uid uint32                         Override the uid field set by the filesystem. (default 502)
+      --umask int                          Override the permission bits set by the filesystem. (default 2)
+      --user string                        User name for authentication.
+      --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
+      --vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
+      --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+      --vfs-read-chunk-size int            Read the source objects in chunks. (default 128M)
+      --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off)
+

rclone serve restic

+

Serve the remote for restic's REST API.

+

Synopsis

+

rclone serve restic implements restic's REST backend API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly.

+

Restic is a command line program for doing backups.

+

The server will log errors. Use -v to see access logs.

+

--bwlimit will be respected for file transfers. Use --stats to control the stats printing.

+

Setting up rclone for use by restic

+

First set up a remote for your chosen cloud provider.

+

Once you have set up the remote, check it is working with, for example "rclone lsd remote:". You may have called the remote something other than "remote:" - just substitute whatever you called it in the following instructions.

+

Now start the rclone restic server

+
rclone serve restic -v remote:backup
+

Where you can replace "backup" in the above by whatever path in the remote you wish to use.

+

By default this will serve on "localhost:8080" you can change this with use of the "--addr" flag.

+

You might wish to start this server on boot.

+

Setting up restic to use rclone

+

Now you can follow the restic instructions on setting up restic.

+

Note that you will need restic 0.8.2 or later to interoperate with rclone.

+

For the example above you will want to use "http://localhost:8080/" as the URL for the REST server.

+

For example:

+
$ export RESTIC_REPOSITORY=rest:http://localhost:8080/
+$ export RESTIC_PASSWORD=yourpassword
+$ restic init
+created restic backend 8b1a4b56ae at rest:http://localhost:8080/
+
+Please note that knowledge of your password is required to access
+the repository. Losing your password means that your data is
+irrecoverably lost.
+$ restic backup /path/to/files/to/backup
+scan [/path/to/files/to/backup]
+scanned 189 directories, 312 files in 0:00
+[0:00] 100.00%  38.128 MiB / 38.128 MiB  501 / 501 items  0 errors  ETA 0:00 
+duration: 0:00
+snapshot 45c8fdd8 saved
+

Multiple repositories

+

Note that you can use the endpoint to host multiple repositories. Do this by adding a directory name or path after the URL. Note that these must end with /. Eg

+
$ export RESTIC_REPOSITORY=rest:http://localhost:8080/user1repo/
+# backup user1 stuff
+$ export RESTIC_REPOSITORY=rest:http://localhost:8080/user2repo/
+# backup user2 stuff
+

Server options

+

Use --addr to specify which IP address and port the server should listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.

+

If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.

+

--server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.

+

--max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.

+

Authentication

+

By default this will serve files without needing a login.

+

You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.

+

Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.

+

To create an htpasswd file:

+
touch htpasswd
+htpasswd -B htpasswd user
+htpasswd -B htpasswd anotherUser
+

The password file can be updated while rclone is running.

+

Use --realm to set the authentication realm.

+

SSL/TLS

+

By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.

+

--cert should be a either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.

+
rclone serve restic remote:path [flags]
+

Options

+
      --addr string                     IPaddress:Port or :Port to bind server to. (default "localhost:8080")
+      --append-only                     disallow deletion of repository data
+      --cert string                     SSL PEM key (concatenation of certificate and CA certificate)
+      --client-ca string                Client certificate authority to verify clients with
+  -h, --help                            help for restic
+      --htpasswd string                 htpasswd file - if not provided no authentication is done
+      --key string                      SSL PEM Private key
+      --max-header-bytes int            Maximum size of request header (default 4096)
+      --pass string                     Password for authentication.
+      --realm string                    realm for authentication (default "rclone")
+      --server-read-timeout duration    Timeout for server reading data (default 1h0m0s)
+      --server-write-timeout duration   Timeout for server writing data (default 1h0m0s)
+      --stdio                           run an HTTP2 server on stdin/stdout
+      --user string                     User name for authentication.
+

rclone serve webdav

+

Serve remote:path over webdav.

+

Synopsis

+

rclone serve webdav implements a basic webdav server to serve the remote over HTTP via the webdav protocol. This can be viewed with a webdav client or you can make a remote of type webdav to read and write it.

+

Webdav options

+

--etag-hash

+

This controls the ETag header. Without this flag the ETag will be based on the ModTime and Size of the object.

+

If this flag is set to "auto" then rclone will choose the first supported hash on the backend or you can use a named hash such as "MD5" or "SHA-1".

+

Use "rclone hashsum" to see the full list.

+

Server options

+

Use --addr to specify which IP address and port the server should listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.

+

If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.

+

--server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.

+

--max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.

+

Authentication

+

By default this will serve files without needing a login.

+

You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.

+

Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.

+

To create an htpasswd file:

+
touch htpasswd
+htpasswd -B htpasswd user
+htpasswd -B htpasswd anotherUser
+

The password file can be updated while rclone is running.

+

Use --realm to set the authentication realm.

+

SSL/TLS

+

By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.

+

--cert should be a either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.

+

Directory Cache

+

Using the --dir-cache-time flag, you can set how long a directory should be considered up to date and not refreshed from the backend. Changes made locally in the mount may appear immediately or invalidate the cache. However, changes done on the remote will only be picked up once the cache expires.

+

Alternatively, you can send a SIGHUP signal to rclone for it to flush all directory caches, regardless of how old they are. Assuming only one rclone instance is running, you can reset the cache like this:

+
kill -SIGHUP $(pidof rclone)
+

If you configure rclone with a remote control then you can use rclone rc to flush the whole directory cache:

+
rclone rc vfs/forget
+

Or individual files or directories:

+
rclone rc vfs/forget file=path/to/file dir=path/to/dir
+

File Buffering

+

The --buffer-size flag determines the amount of memory, that will be used to buffer data in advance.

+

Each open file descriptor will try to keep the specified amount of data in memory at all times. The buffered data is bound to one file descriptor and won't be shared between multiple open file descriptors of the same file.

+

This flag is a upper limit for the used memory per file descriptor. The buffer will only use memory for data that is downloaded but not not yet read. If the buffer is empty, only a small amount of memory will be used. The maximum memory used by rclone for buffering can be up to --buffer-size * open files.

+

File Caching

+

NB File caching is EXPERIMENTAL - use with care!

+

These flags control the VFS file caching options. The VFS layer is used by rclone mount to make a cloud storage system work more like a normal file system.

+

You'll need to enable VFS caching if you want, for example, to read and write simultaneously to a file. See below for more details.

+

Note that the VFS cache works in addition to the cache backend and you may find that you need one or the other or both.

+
--cache-dir string                   Directory rclone will use for caching.
+--vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
+--vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
+--vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+

If run with -vv rclone will print the location of the file cache. The files are stored in the user cache file area which is OS dependent but can be controlled with --cache-dir or setting the appropriate environment variable.

+

The cache has 4 different modes selected by --vfs-cache-mode. The higher the cache mode the more compatible rclone becomes at the cost of using disk space.

+

Note that files are written back to the remote only when they are closed so if rclone is quit or dies with open files then these won't get written back to the remote. However they will still be in the on disk cache.

+

--vfs-cache-mode off

+

In this mode the cache will read directly from the remote and write directly to the remote without caching anything on disk.

+

This will mean some operations are not possible

+ +

--vfs-cache-mode minimal

+

This is very similar to "off" except that files opened for read AND write will be buffered to disks. This means that files opened for write will be a lot more compatible, but uses the minimal disk space.

+

These operations are not possible

+ +

--vfs-cache-mode writes

+

In this mode files opened for read only are still read directly from the remote, write only and read/write files are buffered to disk first.

+

This mode should support all normal file system operations.

+

If an upload fails it will be retried up to --low-level-retries times.

+

--vfs-cache-mode full

+

In this mode all reads and writes are buffered to and from disk. When a file is opened for read it will be downloaded in its entirety first.

+

This may be appropriate for your needs, or you may prefer to look at the cache backend which does a much more sophisticated job of caching, including caching directory hierarchies and chunks of files.

+

In this mode, unlike the others, when a file is written to the disk, it will be kept on the disk after it is written to the remote. It will be purged on a schedule according to --vfs-cache-max-age.

+

This mode should support all normal file system operations.

+

If an upload or download fails it will be retried up to --low-level-retries times.

+
rclone serve webdav remote:path [flags]
+

Options

+
      --addr string                        IPaddress:Port or :Port to bind server to. (default "localhost:8080")
+      --cert string                        SSL PEM key (concatenation of certificate and CA certificate)
+      --client-ca string                   Client certificate authority to verify clients with
+      --dir-cache-time duration            Time to cache directory entries for. (default 5m0s)
+      --etag-hash string                   Which hash to use for the ETag, or auto or blank for off
+      --gid uint32                         Override the gid field set by the filesystem. (default 502)
+  -h, --help                               help for webdav
+      --htpasswd string                    htpasswd file - if not provided no authentication is done
+      --key string                         SSL PEM Private key
+      --max-header-bytes int               Maximum size of request header (default 4096)
+      --no-checksum                        Don't compare checksums on up/download.
+      --no-modtime                         Don't read/write the modification time (can speed things up).
+      --no-seek                            Don't allow seeking in files.
+      --pass string                        Password for authentication.
+      --poll-interval duration             Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable. (default 1m0s)
+      --read-only                          Mount read-only.
+      --realm string                       realm for authentication (default "rclone")
+      --server-read-timeout duration       Timeout for server reading data (default 1h0m0s)
+      --server-write-timeout duration      Timeout for server writing data (default 1h0m0s)
+      --uid uint32                         Override the uid field set by the filesystem. (default 502)
+      --umask int                          Override the permission bits set by the filesystem. (default 2)
+      --user string                        User name for authentication.
+      --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
+      --vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
+      --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+      --vfs-read-chunk-size int            Read the source objects in chunks. (default 128M)
+      --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off)
+

rclone touch

+

Create new file or change file modification time.

+

Synopsis

+

Create new file or change file modification time.

+
rclone touch remote:path [flags]
+

Options

+
  -h, --help               help for touch
+  -C, --no-create          Do not create the file if it does not exist.
+  -t, --timestamp string   Change the modification times to the specified time instead of the current time of day. The argument is of the form 'YYMMDD' (ex. 17.10.30) or 'YYYY-MM-DDTHH:MM:SS' (ex. 2006-01-02T15:04:05)
+

rclone tree

+

List the contents of the remote in a tree like fashion.

+

Synopsis

+

rclone tree lists the contents of a remote in a similar way to the unix tree command.

+

For example

+
$ rclone tree remote:path
+/
+├── file1
+├── file2
+├── file3
+└── subdir
+    ├── file4
+    └── file5
+
+1 directories, 5 files
+

You can use any of the filtering options with the tree command (eg --include and --exclude). You can also use --fast-list.

+

The tree command has many options for controlling the listing which are compatible with the tree command. Note that not all of them have short options as they conflict with rclone's short options.

+
rclone tree remote:path [flags]
+

Options

+
  -a, --all             All files are listed (list . files too).
+  -C, --color           Turn colorization on always.
+  -d, --dirs-only       List directories only.
+      --dirsfirst       List directories before files (-U disables).
+      --full-path       Print the full path prefix for each file.
+  -h, --help            help for tree
+      --human           Print the size in a more human readable way.
+      --level int       Descend only level directories deep.
+  -D, --modtime         Print the date of last modification.
+  -i, --noindent        Don't print indentation lines.
+      --noreport        Turn off file/directory count at end of tree listing.
+  -o, --output string   Output to file instead of stdout.
+  -p, --protections     Print the protections for each file.
+  -Q, --quote           Quote filenames with double quotes.
+  -s, --size            Print the size in bytes of each file.
+      --sort string     Select sort: name,version,size,mtime,ctime.
+      --sort-ctime      Sort files by last status change time.
+  -t, --sort-modtime    Sort files by last modification time.
+  -r, --sort-reverse    Reverse the order of the sort.
+  -U, --unsorted        Leave files unsorted.
+      --version         Sort files alphanumerically by version.
+

Copying single files

+

rclone normally syncs or copies directories. However, if the source remote points to a file, rclone will just copy that file. The destination remote must point to a directory - rclone will give the error Failed to create file system for "remote:file": is a file not a directory if it isn't.

+

For example, suppose you have a remote with a file in called test.jpg, then you could copy just that file like this

+
rclone copy remote:test.jpg /tmp/download
+

The file test.jpg will be placed inside /tmp/download.

+

This is equivalent to specifying

+
rclone copy --files-from /tmp/files remote: /tmp/download
+

Where /tmp/files contains the single line

+
test.jpg
+

It is recommended to use copy when copying individual files, not sync. They have pretty much the same effect but copy will use a lot less memory.

+

Syntax of remote paths

+

The syntax of the paths passed to the rclone command are as follows.

+

/path/to/dir

+

This refers to the local file system.

+

On Windows only \ may be used instead of / in local paths only, non local paths must use /.

+

These paths needn't start with a leading / - if they don't then they will be relative to the current directory.

+

remote:path/to/dir

+

This refers to a directory path/to/dir on remote: as defined in the config file (configured with rclone config).

+

remote:/path/to/dir

+

On most backends this is refers to the same directory as remote:path/to/dir and that format should be preferred. On a very small number of remotes (FTP, SFTP, Dropbox for business) this will refer to a different directory. On these, paths without a leading / will refer to your "home" directory and paths with a leading / will refer to the root.

+

:backend:path/to/dir

+

This is an advanced form for creating remotes on the fly. backend should be the name or prefix of a backend (the type in the config file) and all the configuration for the backend should be provided on the command line (or in environment variables).

+

Eg

+
rclone lsd --http-url https://pub.rclone.org :http:
+

Which lists all the directories in pub.rclone.org.

+

Quoting and the shell

+

When you are typing commands to your computer you are using something called the command line shell. This interprets various characters in an OS specific way.

+

Here are some gotchas which may help users unfamiliar with the shell rules

+

Linux / OSX

+

If your names have spaces or shell metacharacters (eg *, ?, $, ', " etc) then you must quote them. Use single quotes ' by default.

+
rclone copy 'Important files?' remote:backup
+

If you want to send a ' you will need to use ", eg

+
rclone copy "O'Reilly Reviews" remote:backup
+

The rules for quoting metacharacters are complicated and if you want the full details you'll have to consult the manual page for your shell.

+

Windows

+

If your names have spaces in you need to put them in ", eg

+
rclone copy "E:\folder name\folder name\folder name" remote:backup
+

If you are using the root directory on its own then don't quote it (see #464 for why), eg

+
rclone copy E:\ remote:backup
+

Copying files or directories with : in the names

+

rclone uses : to mark a remote name. This is, however, a valid filename component in non-Windows OSes. The remote name parser will only search for a : up to the first / so if you need to act on a file or directory like this then use the full path starting with a /, or use ./ as a current directory prefix.

+

So to sync a directory called sync:me to a remote called remote: use

+
rclone sync ./sync:me remote:path
+

or

+
rclone sync /full/path/to/sync:me remote:path
+

Server Side Copy

+

Most remotes (but not all - see the overview) support server side copy.

+

This means if you want to copy one folder to another then rclone won't download all the files and re-upload them; it will instruct the server to copy them in place.

+

Eg

+
rclone copy s3:oldbucket s3:newbucket
+

Will copy the contents of oldbucket to newbucket without downloading and re-uploading.

+

Remotes which don't support server side copy will download and re-upload in this case.

+

Server side copies are used with sync and copy and will be identified in the log when using the -v flag. The move command may also use them if remote doesn't support server side move directly. This is done by issuing a server side copy then a delete which is much quicker than a download and re-upload.

+

Server side copies will only be attempted if the remote names are the same.

+

This can be used when scripting to make aged backups efficiently, eg

+
rclone sync remote:current-backup remote:previous-backup
+rclone sync /path/to/files remote:current-backup
+

Options

+

Rclone has a number of options to control its behaviour.

+

Options which use TIME use the go time parser. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".

+

Options which use SIZE use kByte by default. However, a suffix of b for bytes, k for kBytes, M for MBytes, G for GBytes, T for TBytes and P for PBytes may be used. These are the binary units, eg 1, 2**10, 2**20, 2**30 respectively.

+

--backup-dir=DIR

+

When using sync, copy or move any files which would have been overwritten or deleted are moved in their original hierarchy into this directory.

+

If --suffix is set, then the moved files will have the suffix added to them. If there is a file with the same path (after the suffix has been added) in DIR, then it will be overwritten.

+

The remote in use must support server side move or copy and you must use the same remote as the destination of the sync. The backup directory must not overlap the destination directory.

+

For example

+
rclone sync /path/to/local remote:current --backup-dir remote:old
+

will sync /path/to/local to remote:current, but for any files which would have been updated or deleted will be stored in remote:old.

+

If running rclone from a script you might want to use today's date as the directory name passed to --backup-dir to store the old files, or you might want to pass --suffix with today's date.

+

--bind string

+

Local address to bind to for outgoing connections. This can be an IPv4 address (1.2.3.4), an IPv6 address (1234::789A) or host name. If the host name doesn't resolve or resolves to more than one IP address it will give an error.

+

--bwlimit=BANDWIDTH_SPEC

+

This option controls the bandwidth limit. Limits can be specified in two ways: As a single limit, or as a timetable.

+

Single limits last for the duration of the session. To use a single limit, specify the desired bandwidth in kBytes/s, or use a suffix b|k|M|G. The default is 0 which means to not limit bandwidth.

+

For example, to limit bandwidth usage to 10 MBytes/s use --bwlimit 10M

+

It is also possible to specify a "timetable" of limits, which will cause certain limits to be applied at certain times. To specify a timetable, format your entries as "WEEKDAY-HH:MM,BANDWIDTH WEEKDAY-HH:MM,BANDWIDTH..." where: WEEKDAY is optional element. It could be writen as whole world or only using 3 first characters. HH:MM is an hour from 00:00 to 23:59.

+

An example of a typical timetable to avoid link saturation during daytime working hours could be:

+

--bwlimit "08:00,512 12:00,10M 13:00,512 18:00,30M 23:00,off"

+

In this example, the transfer bandwidth will be every day set to 512kBytes/sec at 8am. At noon, it will raise to 10Mbytes/s, and drop back to 512kBytes/sec at 1pm. At 6pm, the bandwidth limit will be set to 30MBytes/s, and at 11pm it will be completely disabled (full speed). Anything between 11pm and 8am will remain unlimited.

+

An example of timetable with WEEKDAY could be:

+

--bwlimit "Mon-00:00,512 Fri-23:59,10M Sat-10:00,1M Sun-20:00,off"

+

It mean that, the transfer bandwidh will be set to 512kBytes/sec on Monday. It will raise to 10Mbytes/s before the end of Friday. At 10:00 on Sunday it will be set to 1Mbyte/s. From 20:00 at Sunday will be unlimited.

+

Timeslots without weekday are extended to whole week. So this one example:

+

--bwlimit "Mon-00:00,512 12:00,1M Sun-20:00,off"

+

Is equal to this:

+

--bwlimit "Mon-00:00,512Mon-12:00,1M Tue-12:00,1M Wed-12:00,1M Thu-12:00,1M Fri-12:00,1M Sat-12:00,1M Sun-12:00,1M Sun-20:00,off"

+

Bandwidth limits only apply to the data transfer. They don't apply to the bandwidth of the directory listings etc.

+

Note that the units are Bytes/s, not Bits/s. Typically connections are measured in Bits/s - to convert divide by 8. For example, let's say you have a 10 Mbit/s connection and you wish rclone to use half of it - 5 Mbit/s. This is 5/8 = 0.625MByte/s so you would use a --bwlimit 0.625M parameter for rclone.

+

On Unix systems (Linux, MacOS, …) the bandwidth limiter can be toggled by sending a SIGUSR2 signal to rclone. This allows to remove the limitations of a long running rclone transfer and to restore it back to the value specified with --bwlimit quickly when needed. Assuming there is only one rclone instance running, you can toggle the limiter like this:

+
kill -SIGUSR2 $(pidof rclone)
+

If you configure rclone with a remote control then you can use change the bwlimit dynamically:

+
rclone rc core/bwlimit rate=1M
+

--buffer-size=SIZE

+

Use this sized buffer to speed up file transfers. Each --transfer will use this much memory for buffering.

+

When using mount or cmount each open file descriptor will use this much memory for buffering. See the mount documentation for more details.

+

Set to 0 to disable the buffering for the minimum memory usage.

+

--checkers=N

+

The number of checkers to run in parallel. Checkers do the equality checking of files during a sync. For some storage systems (eg S3, Swift, Dropbox) this can take a significant amount of time so they are run in parallel.

+

The default is to run 8 checkers in parallel.

+

-c, --checksum

+

Normally rclone will look at modification time and size of files to see if they are equal. If you set this flag then rclone will check the file hash and size to determine if files are equal.

+

This is useful when the remote doesn't support setting modified time and a more accurate sync is desired than just checking the file size.

+

This is very useful when transferring between remotes which store the same hash type on the object, eg Drive and Swift. For details of which remotes support which hash type see the table in the overview section.

+

Eg rclone --checksum sync s3:/bucket swift:/bucket would run much quicker than without the --checksum flag.

+

When using this flag, rclone won't update mtimes of remote files if they are incorrect as it would normally.

+

--config=CONFIG_FILE

+

Specify the location of the rclone config file.

+

Normally the config file is in your home directory as a file called .config/rclone/rclone.conf (or .rclone.conf if created with an older version). If $XDG_CONFIG_HOME is set it will be at $XDG_CONFIG_HOME/rclone/rclone.conf

+

If you run rclone -h and look at the help for the --config option you will see where the default location is for you.

+

Use this flag to override the config location, eg rclone --config=".myconfig" .config.

+

--contimeout=TIME

+

Set the connection timeout. This should be in go time format which looks like 5s for 5 seconds, 10m for 10 minutes, or 3h30m.

+

The connection timeout is the amount of time rclone will wait for a connection to go through to a remote object storage system. It is 1m by default.

+

--dedupe-mode MODE

+

Mode to run dedupe command in. One of interactive, skip, first, newest, oldest, rename. The default is interactive. See the dedupe command for more information as to what these options mean.

+

--disable FEATURE,FEATURE,...

+

This disables a comma separated list of optional features. For example to disable server side move and server side copy use:

+
--disable move,copy
+

The features can be put in in any case.

+

To see a list of which features can be disabled use:

+
--disable help
+

See the overview features and optional features to get an idea of which feature does what.

+

This flag can be useful for debugging and in exceptional circumstances (eg Google Drive limiting the total volume of Server Side Copies to 100GB/day).

+

-n, --dry-run

+

Do a trial run with no permanent changes. Use this to see what rclone would do without actually doing it. Useful when setting up the sync command which deletes files in the destination.

+

--ignore-checksum

+

Normally rclone will check that the checksums of transferred files match, and give an error "corrupted on transfer" if they don't.

+

You can use this option to skip that check. You should only use it if you have had the "corrupted on transfer" error message and you are sure you might want to transfer potentially corrupted data.

+

--ignore-existing

+

Using this option will make rclone unconditionally skip all files that exist on the destination, no matter the content of these files.

+

While this isn't a generally recommended option, it can be useful in cases where your files change due to encryption. However, it cannot correct partial transfers in case a transfer was interrupted.

+

--ignore-size

+

Normally rclone will look at modification time and size of files to see if they are equal. If you set this flag then rclone will check only the modification time. If --checksum is set then it only checks the checksum.

+

It will also cause rclone to skip verifying the sizes are the same after transfer.

+

This can be useful for transferring files to and from OneDrive which occasionally misreports the size of image files (see #399 for more info).

+

-I, --ignore-times

+

Using this option will cause rclone to unconditionally upload all files regardless of the state of files on the destination.

+

Normally rclone would skip any files that have the same modification time and are the same size (or have the same checksum if using --checksum).

+

--immutable

+

Treat source and destination files as immutable and disallow modification.

+

With this option set, files will be created and deleted as requested, but existing files will never be updated. If an existing file does not match between the source and destination, rclone will give the error Source and destination exist but do not match: immutable file modified.

+

Note that only commands which transfer files (e.g. sync, copy, move) are affected by this behavior, and only modification is disallowed. Files may still be deleted explicitly (e.g. delete, purge) or implicitly (e.g. sync, move). Use copy --immutable if it is desired to avoid deletion as well as modification.

+

This can be useful as an additional layer of protection for immutable or append-only data sets (notably backup archives), where modification implies corruption and should not be propagated.

+

--leave-root

+

During rmdirs it will not remove root directory, even if it's empty.

+

--log-file=FILE

+

Log all of rclone's output to FILE. This is not active by default. This can be useful for tracking down problems with syncs in combination with the -v flag. See the Logging section for more info.

+

Note that if you are using the logrotate program to manage rclone's logs, then you should use the copytruncate option as rclone doesn't have a signal to rotate logs.

+

--log-level LEVEL

+

This sets the log level for rclone. The default log level is NOTICE.

+

DEBUG is equivalent to -vv. It outputs lots of debug info - useful for bug reports and really finding out what rclone is doing.

+

INFO is equivalent to -v. It outputs information about each transfer and prints stats once a minute by default.

+

NOTICE is the default log level if no logging flags are supplied. It outputs very little when things are working normally. It outputs warnings and significant events.

+

ERROR is equivalent to -q. It only outputs error messages.

+

--low-level-retries NUMBER

+

This controls the number of low level retries rclone does.

+

A low level retry is used to retry a failing operation - typically one HTTP request. This might be uploading a chunk of a big file for example. You will see low level retries in the log with the -v flag.

+

This shouldn't need to be changed from the default in normal operations. However, if you get a lot of low level retries you may wish to reduce the value so rclone moves on to a high level retry (see the --retries flag) quicker.

+

Disable low level retries with --low-level-retries 1.

+

--max-backlog=N

+

This is the maximum allowable backlog of files in a sync/copy/move queued for being checked or transferred.

+

This can be set arbitrarily large. It will only use memory when the queue is in use. Note that it will use in the order of N kB of memory when the backlog is in use.

+

Setting this large allows rclone to calculate how many files are pending more accurately and give a more accurate estimated finish time.

+

Setting this small will make rclone more synchronous to the listings of the remote which may be desirable.

+

--max-delete=N

+

This tells rclone not to delete more than N files. If that limit is exceeded then a fatal error will be generated and rclone will stop the operation in progress.

+

--max-depth=N

+

This modifies the recursion depth for all the commands except purge.

+

So if you do rclone --max-depth 1 ls remote:path you will see only the files in the top level directory. Using --max-depth 2 means you will see all the files in first two directory levels and so on.

+

For historical reasons the lsd command defaults to using a --max-depth of 1 - you can override this with the command line flag.

+

You can use this command to disable recursion (with --max-depth 1).

+

Note that if you use this with sync and --delete-excluded the files not recursed through are considered excluded and will be deleted on the destination. Test first with --dry-run if you are not sure what will happen.

+

--max-transfer=SIZE

+

Rclone will stop transferring when it has reached the size specified. Defaults to off.

+

When the limit is reached all transfers will stop immediately.

+

Rclone will exit with exit code 8 if the transfer limit is reached.

+

--modify-window=TIME

+

When checking whether a file has been modified, this is the maximum allowed time difference that a file can have and still be considered equivalent.

+

The default is 1ns unless this is overridden by a remote. For example OS X only stores modification times to the nearest second so if you are reading and writing to an OS X filing system this will be 1s by default.

+

This command line flag allows you to override that computed default.

+

--no-gzip-encoding

+

Don't set Accept-Encoding: gzip. This means that rclone won't ask the server for compressed files automatically. Useful if you've set the server to return files with Content-Encoding: gzip but you uploaded compressed files.

+

There is no need to set this in normal operation, and doing so will decrease the network transfer efficiency of rclone.

+

--no-update-modtime

+

When using this flag, rclone won't update modification times of remote files if they are incorrect as it would normally.

+

This can be used if the remote is being synced with another tool also (eg the Google Drive client).

+

--P, --progress

+

This flag makes rclone update the stats in a static block in the terminal providing a realtime overview of the transfer.

+

Any log messages will scroll above the static block. Log messages will push the static block down to the bottom of the terminal where it will stay.

+

Normally this is updated every 500mS but this period can be overridden with the --stats flag.

+

This can be used with the --stats-one-line flag for a simpler display.

+

-q, --quiet

+

Normally rclone outputs stats and a completion message. If you set this flag it will make as little output as possible.

+

--retries int

+

Retry the entire sync if it fails this many times it fails (default 3).

+

Some remotes can be unreliable and a few retries help pick up the files which didn't get transferred because of errors.

+

Disable retries with --retries 1.

+

--retries-sleep=TIME

+

This sets the interval between each retry specified by --retries

+

The default is 0. Use 0 to disable.

+

--size-only

+

Normally rclone will look at modification time and size of files to see if they are equal. If you set this flag then rclone will check only the size.

+

This can be useful transferring files from Dropbox which have been modified by the desktop sync client which doesn't set checksums of modification times in the same way as rclone.

+

--stats=TIME

+

Commands which transfer data (sync, copy, copyto, move, moveto) will print data transfer stats at regular intervals to show their progress.

+

This sets the interval.

+

The default is 1m. Use 0 to disable.

+

If you set the stats interval then all commands can show stats. This can be useful when running other commands, check or mount for example.

+

Stats are logged at INFO level by default which means they won't show at default log level NOTICE. Use --stats-log-level NOTICE or -v to make them show. See the Logging section for more info on log levels.

+

Note that on macOS you can send a SIGINFO (which is normally ctrl-T in the terminal) to make the stats print immediately.

+

--stats-file-name-length integer

+

By default, the --stats output will truncate file names and paths longer than 40 characters. This is equivalent to providing --stats-file-name-length 40. Use --stats-file-name-length 0 to disable any truncation of file names printed by stats.

+

--stats-log-level string

+

Log level to show --stats output at. This can be DEBUG, INFO, NOTICE, or ERROR. The default is INFO. This means at the default level of logging which is NOTICE the stats won't show - if you want them to then use --stats-log-level NOTICE. See the Logging section for more info on log levels.

+

--stats-one-line

+

When this is specified, rclone condenses the stats into a single line showing the most important stats only.

+

--stats-unit=bits|bytes

+

By default, data transfer rates will be printed in bytes/second.

+

This option allows the data rate to be printed in bits/second.

+

Data transfer volume will still be reported in bytes.

+

The rate is reported as a binary unit, not SI unit. So 1 Mbit/s equals 1,048,576 bits/s and not 1,000,000 bits/s.

+

The default is bytes.

+

--suffix=SUFFIX

+

This is for use with --backup-dir only. If this isn't set then --backup-dir will move files with their original name. If it is set then the files will have SUFFIX added on to them.

+

See --backup-dir for more info.

+

--syslog

+

On capable OSes (not Windows or Plan9) send all log output to syslog.

+

This can be useful for running rclone in a script or rclone mount.

+

--syslog-facility string

+

If using --syslog this sets the syslog facility (eg KERN, USER). See man syslog for a list of possible facilities. The default facility is DAEMON.

+

--tpslimit float

+

Limit HTTP transactions per second to this. Default is 0 which is used to mean unlimited transactions per second.

+

For example to limit rclone to 10 HTTP transactions per second use --tpslimit 10, or to 1 transaction every 2 seconds use --tpslimit 0.5.

+

Use this when the number of transactions per second from rclone is causing a problem with the cloud storage provider (eg getting you banned or rate limited).

+

This can be very useful for rclone mount to control the behaviour of applications using it.

+

See also --tpslimit-burst.

+

--tpslimit-burst int

+

Max burst of transactions for --tpslimit. (default 1)

+

Normally --tpslimit will do exactly the number of transaction per second specified. However if you supply --tps-burst then rclone can save up some transactions from when it was idle giving a burst of up to the parameter supplied.

+

For example if you provide --tpslimit-burst 10 then if rclone has been idle for more than 10*--tpslimit then it can do 10 transactions very quickly before they are limited again.

+

This may be used to increase performance of --tpslimit without changing the long term average number of transactions per second.

+

--track-renames

+

By default, rclone doesn't keep track of renamed files, so if you rename a file locally then sync it to a remote, rclone will delete the old file on the remote and upload a new copy.

+

If you use this flag, and the remote supports server side copy or server side move, and the source and destination have a compatible hash, then this will track renames during sync operations and perform renaming server-side.

+

Files will be matched by size and hash - if both match then a rename will be considered.

+

If the destination does not support server-side copy or move, rclone will fall back to the default behaviour and log an error level message to the console.

+

Note that --track-renames uses extra memory to keep track of all the rename candidates.

+

Note also that --track-renames is incompatible with --delete-before and will select --delete-after instead of --delete-during.

+

--delete-(before,during,after)

+

This option allows you to specify when files on your destination are deleted when you sync folders.

+

Specifying the value --delete-before will delete all files present on the destination, but not on the source before starting the transfer of any new or updated files. This uses two passes through the file systems, one for the deletions and one for the copies.

+

Specifying --delete-during will delete files while checking and uploading files. This is the fastest option and uses the least memory.

+

Specifying --delete-after (the default value) will delay deletion of files until all new/updated files have been successfully transferred. The files to be deleted are collected in the copy pass then deleted after the copy pass has completed successfully. The files to be deleted are held in memory so this mode may use more memory. This is the safest mode as it will only delete files if there have been no errors subsequent to that. If there have been errors before the deletions start then you will get the message not deleting files as there were IO errors.

+

--fast-list

+

When doing anything which involves a directory listing (eg sync, copy, ls - in fact nearly every command), rclone normally lists a directory and processes it before using more directory lists to process any subdirectories. This can be parallelised and works very quickly using the least amount of memory.

+

However, some remotes have a way of listing all files beneath a directory in one (or a small number) of transactions. These tend to be the bucket based remotes (eg S3, B2, GCS, Swift, Hubic).

+

If you use the --fast-list flag then rclone will use this method for listing directories. This will have the following consequences for the listing:

+ +

rclone should always give identical results with and without --fast-list.

+

If you pay for transactions and can fit your entire sync listing into memory then --fast-list is recommended. If you have a very big sync to do then don't use --fast-list otherwise you will run out of memory.

+

If you use --fast-list on a remote which doesn't support it, then rclone will just ignore it.

+

--timeout=TIME

+

This sets the IO idle timeout. If a transfer has started but then becomes idle for this long it is considered broken and disconnected.

+

The default is 5m. Set to 0 to disable.

+

--transfers=N

+

The number of file transfers to run in parallel. It can sometimes be useful to set this to a smaller number if the remote is giving a lot of timeouts or bigger if you have lots of bandwidth and a fast remote.

+

The default is to run 4 file transfers in parallel.

+

-u, --update

+

This forces rclone to skip any files which exist on the destination and have a modified time that is newer than the source file.

+

If an existing destination file has a modification time equal (within the computed modify window precision) to the source file's, it will be updated if the sizes are different.

+

On remotes which don't support mod time directly the time checked will be the uploaded time. This means that if uploading to one of these remotes, rclone will skip any files which exist on the destination and have an uploaded time that is newer than the modification time of the source file.

+

This can be useful when transferring to a remote which doesn't support mod times directly as it is more accurate than a --size-only check and faster than using --checksum.

+

--use-server-modtime

+

Some object-store backends (e.g, Swift, S3) do not preserve file modification times (modtime). On these backends, rclone stores the original modtime as additional metadata on the object. By default it will make an API call to retrieve the metadata when the modtime is needed by an operation.

+

Use this flag to disable the extra API call and rely instead on the server's modified time. In cases such as a local to remote sync, knowing the local file is newer than the time it was last uploaded to the remote is sufficient. In those cases, this flag can speed up the process and reduce the number of API calls necessary.

+

-v, -vv, --verbose

+

With -v rclone will tell you about each file that is transferred and a small number of significant events.

+

With -vv rclone will become very verbose telling you about every file it considers and transfers. Please send bug reports with a log with this setting.

+

-V, --version

+

Prints the version number

+

Configuration Encryption

+

Your configuration file contains information for logging in to your cloud services. This means that you should keep your .rclone.conf file in a secure location.

+

If you are in an environment where that isn't possible, you can add a password to your configuration. This means that you will have to enter the password every time you start rclone.

+

To add a password to your rclone configuration, execute rclone config.

+
>rclone config
+Current remotes:
+
+e) Edit existing remote
+n) New remote
+d) Delete remote
+s) Set configuration password
+q) Quit config
+e/n/d/s/q>
+

Go into s, Set configuration password:

+
e/n/d/s/q> s
+Your configuration is not encrypted.
+If you add a password, you will protect your login information to cloud services.
+a) Add Password
+q) Quit to main menu
+a/q> a
+Enter NEW configuration password:
+password:
+Confirm NEW password:
+password:
+Password set
+Your configuration is encrypted.
+c) Change Password
+u) Unencrypt configuration
+q) Quit to main menu
+c/u/q>
+

Your configuration is now encrypted, and every time you start rclone you will now be asked for the password. In the same menu, you can change the password or completely remove encryption from your configuration.

+

There is no way to recover the configuration if you lose your password.

+

rclone uses nacl secretbox which in turn uses XSalsa20 and Poly1305 to encrypt and authenticate your configuration with secret-key cryptography. The password is SHA-256 hashed, which produces the key for secretbox. The hashed password is not stored.

+

While this provides very good security, we do not recommend storing your encrypted rclone configuration in public if it contains sensitive information, maybe except if you use a very strong password.

+

If it is safe in your environment, you can set the RCLONE_CONFIG_PASS environment variable to contain your password, in which case it will be used for decrypting the configuration.

+

You can set this for a session from a script. For unix like systems save this to a file called set-rclone-password:

+
#!/bin/echo Source this file don't run it
+
+read -s RCLONE_CONFIG_PASS
+export RCLONE_CONFIG_PASS
+

Then source the file when you want to use it. From the shell you would do source set-rclone-password. It will then ask you for the password and set it in the environment variable.

+

If you are running rclone inside a script, you might want to disable password prompts. To do that, pass the parameter --ask-password=false to rclone. This will make rclone fail instead of asking for a password if RCLONE_CONFIG_PASS doesn't contain a valid password.

+

Developer options

+

These options are useful when developing or debugging rclone. There are also some more remote specific options which aren't documented here which are used for testing. These start with remote name eg --drive-test-option - see the docs for the remote in question.

+

--cpuprofile=FILE

+

Write CPU profile to file. This can be analysed with go tool pprof.

+

--dump flag,flag,flag

+

The --dump flag takes a comma separated list of flags to dump info about. These are:

+

--dump headers

+

Dump HTTP headers with Authorization: lines removed. May still contain sensitive info. Can be very verbose. Useful for debugging only.

+

Use --dump auth if you do want the Authorization: headers.

+

--dump bodies

+

Dump HTTP headers and bodies - may contain sensitive info. Can be very verbose. Useful for debugging only.

+

Note that the bodies are buffered in memory so don't use this for enormous files.

+

--dump requests

+

Like --dump bodies but dumps the request bodies and the response headers. Useful for debugging download problems.

+

--dump responses

+

Like --dump bodies but dumps the response bodies and the request headers. Useful for debugging upload problems.

+

--dump auth

+

Dump HTTP headers - will contain sensitive info such as Authorization: headers - use --dump headers to dump without Authorization: headers. Can be very verbose. Useful for debugging only.

+

--dump filters

+

Dump the filters to the output. Useful to see exactly what include and exclude options are filtering on.

+

--dump goroutines

+

This dumps a list of the running go-routines at the end of the command to standard output.

+

--dump openfiles

+

This dumps a list of the open files at the end of the command. It uses the lsof command to do that so you'll need that installed to use it.

+

--memprofile=FILE

+

Write memory profile to file. This can be analysed with go tool pprof.

+

--no-check-certificate=true/false

+

--no-check-certificate controls whether a client verifies the server's certificate chain and host name. If --no-check-certificate is true, TLS accepts any certificate presented by the server and any host name in that certificate. In this mode, TLS is susceptible to man-in-the-middle attacks.

+

This option defaults to false.

+

This should be used only for testing.

+

Filtering

+

For the filtering options

+ +

See the filtering section.

+

Remote control

+

For the remote control options and for instructions on how to remote control rclone

+ +

See the remote control section.

+

Logging

+

rclone has 4 levels of logging, ERROR, NOTICE, INFO and DEBUG.

+

By default, rclone logs to standard error. This means you can redirect standard error and still see the normal output of rclone commands (eg rclone ls).

+

By default, rclone will produce Error and Notice level messages.

+

If you use the -q flag, rclone will only produce Error messages.

+

If you use the -v flag, rclone will produce Error, Notice and Info messages.

+

If you use the -vv flag, rclone will produce Error, Notice, Info and Debug messages.

+

You can also control the log levels with the --log-level flag.

+

If you use the --log-file=FILE option, rclone will redirect Error, Info and Debug messages along with standard error to FILE.

+

If you use the --syslog flag then rclone will log to syslog and the --syslog-facility control which facility it uses.

+

Rclone prefixes all log messages with their level in capitals, eg INFO which makes it easy to grep the log file for different kinds of information.

+

Exit Code

+

If any errors occur during the command execution, rclone will exit with a non-zero exit code. This allows scripts to detect when rclone operations have failed.

+

During the startup phase, rclone will exit immediately if an error is detected in the configuration. There will always be a log message immediately before exiting.

+

When rclone is running it will accumulate errors as it goes along, and only exit with a non-zero exit code if (after retries) there were still failed transfers. For every error counted there will be a high priority log message (visible with -q) showing the message and which file caused the problem. A high priority message is also shown when starting a retry so the user can see that any previous error messages may not be valid after the retry. If rclone has done a retry it will log a high priority message if the retry was successful.

+

List of exit codes

+ +

Environment Variables

+

Rclone can be configured entirely using environment variables. These can be used to set defaults for options or config file entries.

+

Options

+

Every option in rclone can have its default set by environment variable.

+

To find the name of the environment variable, first, take the long option name, strip the leading --, change - to _, make upper case and prepend RCLONE_.

+

For example, to always set --stats 5s, set the environment variable RCLONE_STATS=5s. If you set stats on the command line this will override the environment variable setting.

+

Or to always use the trash in drive --drive-use-trash, set RCLONE_DRIVE_USE_TRASH=true.

+

The same parser is used for the options and the environment variables so they take exactly the same form.

+

Config file

+

You can set defaults for values in the config file on an individual remote basis. If you want to use this feature, you will need to discover the name of the config items that you want. The easiest way is to run through rclone config by hand, then look in the config file to see what the values are (the config file can be found by looking at the help for --config in rclone help).

+

To find the name of the environment variable, you need to set, take RCLONE_CONFIG_ + name of remote + _ + name of config file option and make it all uppercase.

+

For example, to configure an S3 remote named mys3: without a config file (using unix ways of setting environment variables):

+
$ export RCLONE_CONFIG_MYS3_TYPE=s3
+$ export RCLONE_CONFIG_MYS3_ACCESS_KEY_ID=XXX
+$ export RCLONE_CONFIG_MYS3_SECRET_ACCESS_KEY=XXX
+$ rclone lsd MYS3:
+          -1 2016-09-21 12:54:21        -1 my-bucket
+$ rclone listremotes | grep mys3
+mys3:
+

Note that if you want to create a remote using environment variables you must create the ..._TYPE variable as above.

+

Other environment variables

+ +

Configuring rclone on a remote / headless machine

+

Some of the configurations (those involving oauth2) require an Internet connected web browser.

+

If you are trying to set rclone up on a remote or headless box with no browser available on it (eg a NAS or a server in a datacenter) then you will need to use an alternative means of configuration. There are two ways of doing it, described below.

+

Configuring using rclone authorize

+

On the headless box

+
...
+Remote config
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine
+y) Yes
+n) No
+y/n> n
+For this to work, you will need rclone available on a machine that has a web browser available.
+Execute the following on your machine:
+    rclone authorize "amazon cloud drive"
+Then paste the result below:
+result>
+

Then on your main desktop machine

+
rclone authorize "amazon cloud drive"
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+Paste the following into your remote machine --->
+SECRET_TOKEN
+<---End paste
+

Then back to the headless box, paste in the code

+
result> SECRET_TOKEN
+--------------------
+[acd12]
+client_id = 
+client_secret = 
+token = SECRET_TOKEN
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d>
+

Configuring by copying the config file

+

Rclone stores all of its config in a single configuration file. This can easily be copied to configure a remote rclone.

+

So first configure rclone on your desktop machine

+
rclone config
+

to set up the config file.

+

Find the config file by running rclone -h and looking for the help for the --config option

+
$ rclone -h
+[snip]
+      --config="/home/user/.rclone.conf": Config file.
+[snip]
+

Now transfer it to the remote box (scp, cut paste, ftp, sftp etc) and place it in the correct place (use rclone -h on the remote box to find out where).

+

Filtering, includes and excludes

+

Rclone has a sophisticated set of include and exclude rules. Some of these are based on patterns and some on other things like file size.

+

The filters are applied for the copy, sync, move, ls, lsl, md5sum, sha1sum, size, delete and check operations. Note that purge does not obey the filters.

+

Each path as it passes through rclone is matched against the include and exclude rules like --include, --exclude, --include-from, --exclude-from, --filter, or --filter-from. The simplest way to try them out is using the ls command, or --dry-run together with -v.

+

Patterns

+

The patterns used to match files for inclusion or exclusion are based on "file globs" as used by the unix shell.

+

If the pattern starts with a / then it only matches at the top level of the directory tree, relative to the root of the remote (not necessarily the root of the local drive). If it doesn't start with / then it is matched starting at the end of the path, but it will only match a complete path element:

+
file.jpg  - matches "file.jpg"
+          - matches "directory/file.jpg"
+          - doesn't match "afile.jpg"
+          - doesn't match "directory/afile.jpg"
+/file.jpg - matches "file.jpg" in the root directory of the remote
+          - doesn't match "afile.jpg"
+          - doesn't match "directory/file.jpg"
+

Important Note that you must use / in patterns and not \ even if running on Windows.

+

A * matches anything but not a /.

+
*.jpg  - matches "file.jpg"
+       - matches "directory/file.jpg"
+       - doesn't match "file.jpg/something"
+

Use ** to match anything, including slashes (/).

+
dir/** - matches "dir/file.jpg"
+       - matches "dir/dir1/dir2/file.jpg"
+       - doesn't match "directory/file.jpg"
+       - doesn't match "adir/file.jpg"
+

A ? matches any character except a slash /.

+
l?ss  - matches "less"
+      - matches "lass"
+      - doesn't match "floss"
+

A [ and ] together make a a character class, such as [a-z] or [aeiou] or [[:alpha:]]. See the go regexp docs for more info on these.

+
h[ae]llo - matches "hello"
+         - matches "hallo"
+         - doesn't match "hullo"
+

A { and } define a choice between elements. It should contain a comma separated list of patterns, any of which might match. These patterns can contain wildcards.

+
{one,two}_potato - matches "one_potato"
+                 - matches "two_potato"
+                 - doesn't match "three_potato"
+                 - doesn't match "_potato"
+

Special characters can be escaped with a \ before them.

+
\*.jpg       - matches "*.jpg"
+\\.jpg       - matches "\.jpg"
+\[one\].jpg  - matches "[one].jpg"
+

Note also that rclone filter globs can only be used in one of the filter command line flags, not in the specification of the remote, so rclone copy "remote:dir*.jpg" /path/to/dir won't work - what is required is rclone --include "*.jpg" copy remote:dir /path/to/dir

+

Directories

+

Rclone keeps track of directories that could match any file patterns.

+

Eg if you add the include rule

+
/a/*.jpg
+

Rclone will synthesize the directory include rule

+
/a/
+

If you put any rules which end in / then it will only match directories.

+

Directory matches are only used to optimise directory access patterns - you must still match the files that you want to match. Directory matches won't optimise anything on bucket based remotes (eg s3, swift, google compute storage, b2) which don't have a concept of directory.

+

Differences between rsync and rclone patterns

+

Rclone implements bash style {a,b,c} glob matching which rsync doesn't.

+

Rclone always does a wildcard match so \ must always escape a \.

+

How the rules are used

+

Rclone maintains a combined list of include rules and exclude rules.

+

Each file is matched in order, starting from the top, against the rule in the list until it finds a match. The file is then included or excluded according to the rule type.

+

If the matcher fails to find a match after testing against all the entries in the list then the path is included.

+

For example given the following rules, + being include, - being exclude,

+
- secret*.jpg
++ *.jpg
++ *.png
++ file2.avi
+- *
+

This would include

+ +

This would exclude

+ +

A similar process is done on directory entries before recursing into them. This only works on remotes which have a concept of directory (Eg local, google drive, onedrive, amazon drive) and not on bucket based remotes (eg s3, swift, google compute storage, b2).

+

Adding filtering rules

+

Filtering rules are added with the following command line flags.

+

Repeating options

+

You can repeat the following options to add more than one rule of that type.

+ +

Important You should not use --include* together with --exclude*. It may produce different results than you expected. In that case try to use: --filter*.

+

Note that all the options of the same type are processed together in the order above, regardless of what order they were placed on the command line.

+

So all --include options are processed first in the order they appeared on the command line, then all --include-from options etc.

+

To mix up the order includes and excludes, the --filter flag can be used.

+

--exclude - Exclude files matching pattern

+

Add a single exclude rule with --exclude.

+

This flag can be repeated. See above for the order the flags are processed in.

+

Eg --exclude *.bak to exclude all bak files from the sync.

+

--exclude-from - Read exclude patterns from file

+

Add exclude rules from a file.

+

This flag can be repeated. See above for the order the flags are processed in.

+

Prepare a file like this exclude-file.txt

+
# a sample exclude rule file
+*.bak
+file2.jpg
+

Then use as --exclude-from exclude-file.txt. This will sync all files except those ending in bak and file2.jpg.

+

This is useful if you have a lot of rules.

+

--include - Include files matching pattern

+

Add a single include rule with --include.

+

This flag can be repeated. See above for the order the flags are processed in.

+

Eg --include *.{png,jpg} to include all png and jpg files in the backup and no others.

+

This adds an implicit --exclude * at the very end of the filter list. This means you can mix --include and --include-from with the other filters (eg --exclude) but you must include all the files you want in the include statement. If this doesn't provide enough flexibility then you must use --filter-from.

+

--include-from - Read include patterns from file

+

Add include rules from a file.

+

This flag can be repeated. See above for the order the flags are processed in.

+

Prepare a file like this include-file.txt

+
# a sample include rule file
+*.jpg
+*.png
+file2.avi
+

Then use as --include-from include-file.txt. This will sync all jpg, png files and file2.avi.

+

This is useful if you have a lot of rules.

+

This adds an implicit --exclude * at the very end of the filter list. This means you can mix --include and --include-from with the other filters (eg --exclude) but you must include all the files you want in the include statement. If this doesn't provide enough flexibility then you must use --filter-from.

+

--filter - Add a file-filtering rule

+

This can be used to add a single include or exclude rule. Include rules start with + and exclude rules start with -. A special rule called ! can be used to clear the existing rules.

+

This flag can be repeated. See above for the order the flags are processed in.

+

Eg --filter "- *.bak" to exclude all bak files from the sync.

+

--filter-from - Read filtering patterns from a file

+

Add include/exclude rules from a file.

+

This flag can be repeated. See above for the order the flags are processed in.

+

Prepare a file like this filter-file.txt

+
# a sample filter rule file
+- secret*.jpg
++ *.jpg
++ *.png
++ file2.avi
+- /dir/Trash/**
++ /dir/**
+# exclude everything else
+- *
+

Then use as --filter-from filter-file.txt. The rules are processed in the order that they are defined.

+

This example will include all jpg and png files, exclude any files matching secret*.jpg and include file2.avi. It will also include everything in the directory dir at the root of the sync, except dir/Trash which it will exclude. Everything else will be excluded from the sync.

+

--files-from - Read list of source-file names

+

This reads a list of file names from the file passed in and only these files are transferred. The filtering rules are ignored completely if you use this option.

+

This option can be repeated to read from more than one file. These are read in the order that they are placed on the command line.

+

Paths within the --files-from file will be interpreted as starting with the root specified in the command. Leading / characters are ignored.

+

For example, suppose you had files-from.txt with this content:

+
# comment
+file1.jpg
+subdir/file2.jpg
+

You could then use it like this:

+
rclone copy --files-from files-from.txt /home/me/pics remote:pics
+

This will transfer these files only (if they exist)

+
/home/me/pics/file1.jpg        → remote:pics/file1.jpg
+/home/me/pics/subdir/file2.jpg → remote:pics/subdirfile1.jpg
+

To take a more complicated example, let's say you had a few files you want to back up regularly with these absolute paths:

+
/home/user1/important
+/home/user1/dir/file
+/home/user2/stuff
+

To copy these you'd find a common subdirectory - in this case /home and put the remaining files in files-from.txt with or without leading /, eg

+
user1/important
+user1/dir/file
+user2/stuff
+

You could then copy these to a remote like this

+
rclone copy --files-from files-from.txt /home remote:backup
+

The 3 files will arrive in remote:backup with the paths as in the files-from.txt like this:

+
/home/user1/important → remote:backup/user1/important
+/home/user1/dir/file  → remote:backup/user1/dir/file
+/home/user2/stuff     → remote:backup/stuff
+

You could of course choose / as the root too in which case your files-from.txt might look like this.

+
/home/user1/important
+/home/user1/dir/file
+/home/user2/stuff
+

And you would transfer it like this

+
rclone copy --files-from files-from.txt / remote:backup
+

In this case there will be an extra home directory on the remote:

+
/home/user1/important → remote:home/backup/user1/important
+/home/user1/dir/file  → remote:home/backup/user1/dir/file
+/home/user2/stuff     → remote:home/backup/stuff
+

--min-size - Don't transfer any file smaller than this

+

This option controls the minimum size file which will be transferred. This defaults to kBytes but a suffix of k, M, or G can be used.

+

For example --min-size 50k means no files smaller than 50kByte will be transferred.

+

--max-size - Don't transfer any file larger than this

+

This option controls the maximum size file which will be transferred. This defaults to kBytes but a suffix of k, M, or G can be used.

+

For example --max-size 1G means no files larger than 1GByte will be transferred.

+

--max-age - Don't transfer any file older than this

+

This option controls the maximum age of files to transfer. Give in seconds or with a suffix of:

+ +

For example --max-age 2d means no files older than 2 days will be transferred.

+

--min-age - Don't transfer any file younger than this

+

This option controls the minimum age of files to transfer. Give in seconds or with a suffix (see --max-age for list of suffixes)

+

For example --min-age 2d means no files younger than 2 days will be transferred.

+

--delete-excluded - Delete files on dest excluded from sync

+

Important this flag is dangerous - use with --dry-run and -v first.

+

When doing rclone sync this will delete any files which are excluded from the sync on the destination.

+

If for example you did a sync from A to B without the --min-size 50k flag

+
rclone sync A: B:
+

Then you repeated it like this with the --delete-excluded

+
rclone --min-size 50k --delete-excluded sync A: B:
+

This would delete all files on B which are less than 50 kBytes as these are now excluded from the sync.

+

Always test first with --dry-run and -v before using this flag.

+

--dump filters - dump the filters to the output

+

This dumps the defined filters to the output as regular expressions.

+

Useful for debugging.

+

Quoting shell metacharacters

+

The examples above may not work verbatim in your shell as they have shell metacharacters in them (eg *), and may require quoting.

+

Eg linux, OSX

+ +

In Windows the expansion is done by the command not the shell so this should work fine

+ +

Exclude directory based on a file

+

It is possible to exclude a directory based on a file, which is present in this directory. Filename should be specified using the --exclude-if-present flag. This flag has a priority over the other filtering flags.

+

Imagine, you have the following directory structure:

+
dir1/file1
+dir1/dir2/file2
+dir1/dir2/dir3/file3
+dir1/dir2/dir3/.ignore
+

You can exclude dir3 from sync by running the following command:

+
rclone sync --exclude-if-present .ignore dir1 remote:backup
+

Currently only one filename is supported, i.e. --exclude-if-present should not be used multiple times.

+

Remote controlling rclone

+

If rclone is run with the --rc flag then it starts an http server which can be used to remote control rclone.

+

NB this is experimental and everything here is subject to change!

+

Supported parameters

+

--rc

+

Flag to start the http server listen on remote requests

+

--rc-addr=IP

+

IPaddress:Port or :Port to bind server to. (default "localhost:5572")

+

--rc-cert=KEY

+

SSL PEM key (concatenation of certificate and CA certificate)

+

--rc-client-ca=PATH

+

Client certificate authority to verify clients with

+

--rc-htpasswd=PATH

+

htpasswd file - if not provided no authentication is done

+

--rc-key=PATH

+

SSL PEM Private key

+

--rc-max-header-bytes=VALUE

+

Maximum size of request header (default 4096)

+

--rc-user=VALUE

+

User name for authentication.

+

--rc-pass=VALUE

+

Password for authentication.

+

--rc-realm=VALUE

+

Realm for authentication (default "rclone")

+

--rc-server-read-timeout=DURATION

+

Timeout for server reading data (default 1h0m0s)

+

--rc-server-write-timeout=DURATION

+

Timeout for server writing data (default 1h0m0s)

+

Accessing the remote control via the rclone rc command

+

Rclone itself implements the remote control protocol in its rclone rc command.

+

You can use it like this

+
$ rclone rc rc/noop param1=one param2=two
+{
+    "param1": "one",
+    "param2": "two"
+}
+

Run rclone rc on its own to see the help for the installed remote control commands.

+

Supported commands

+ +

cache/expire: Purge a remote from cache

+

Purge a remote from the cache backend. Supports either a directory or a file. Params: - remote = path to remote (required) - withData = true/false to delete cached data (chunks) as well (optional)

+

Eg

+
rclone rc cache/expire remote=path/to/sub/folder/
+rclone rc cache/expire remote=/ withData=true
+

cache/stats: Get cache stats

+

Show statistics for the cache remote.

+

core/bwlimit: Set the bandwidth limit.

+

This sets the bandwidth limit to that passed in.

+

Eg

+
rclone rc core/bwlimit rate=1M
+rclone rc core/bwlimit rate=off
+

The format of the parameter is exactly the same as passed to --bwlimit except only one bandwidth may be specified.

+

core/gc: Runs a garbage collection.

+

This tells the go runtime to do a garbage collection run. It isn't necessary to call this normally, but it can be useful for debugging memory problems.

+

core/memstats: Returns the memory statistics

+

This returns the memory statistics of the running program. What the values mean are explained in the go docs: https://golang.org/pkg/runtime/#MemStats

+

The most interesting values for most people are:

+ +

core/pid: Return PID of current process

+

This returns PID of current process. Useful for stopping rclone process.

+

core/stats: Returns stats about current transfers.

+

This returns all available stats

+
rclone rc core/stats
+

Returns the following values:

+
{
+    "speed": average speed in bytes/sec since start of the process,
+    "bytes": total transferred bytes since the start of the process,
+    "errors": number of errors,
+    "checks": number of checked files,
+    "transfers": number of transferred files,
+    "deletes" : number of deleted files,
+    "elapsedTime": time in seconds since the start of the process,
+    "lastError": last occurred error,
+    "transferring": an array of currently active file transfers:
+        [
+            {
+                "bytes": total transferred bytes for this file,
+                "eta": estimated time in seconds until file transfer completion
+                "name": name of the file,
+                "percentage": progress of the file transfer in percent,
+                "speed": speed in bytes/sec,
+                "speedAvg": speed in bytes/sec as an exponentially weighted moving average,
+                "size": size of the file in bytes
+            }
+        ],
+    "checking": an array of names of currently active file checks
+        []
+}
+

Values for "transferring", "checking" and "lastError" are only assigned if data is available. The value for "eta" is null if an eta cannot be determined.

+

rc/error: This returns an error

+

This returns an error with the input as part of its error string. Useful for testing error handling.

+

rc/list: List all the registered remote control commands

+

This lists all the registered remote control commands as a JSON map in the commands response.

+

rc/noop: Echo the input to the output parameters

+

This echoes the input parameters to the output parameters for testing purposes. It can be used to check that rclone is still alive and to check that parameter passing is working properly.

+

vfs/forget: Forget files or directories in the directory cache.

+

This forgets the paths in the directory cache causing them to be re-read from the remote when needed.

+

If no paths are passed in then it will forget all the paths in the directory cache.

+
rclone rc vfs/forget
+

Otherwise pass files or dirs in as file=path or dir=path. Any parameter key starting with file will forget that file and any starting with dir will forget that dir, eg

+
rclone rc vfs/forget file=hello file2=goodbye dir=home/junk
+

vfs/refresh: Refresh the directory cache.

+

This reads the directories for the specified paths and freshens the directory cache.

+

If no paths are passed in then it will refresh the root directory.

+
rclone rc vfs/refresh
+

Otherwise pass directories in as dir=path. Any parameter key starting with dir will refresh that directory, eg

+
rclone rc vfs/refresh dir=home/junk dir2=data/misc
+

If the parameter recursive=true is given the whole directory tree will get refreshed. This refresh will use --fast-list if enabled.

+ +

Accessing the remote control via HTTP

+

Rclone implements a simple HTTP based protocol.

+

Each endpoint takes an JSON object and returns a JSON object or an error. The JSON objects are essentially a map of string names to values.

+

All calls must made using POST.

+

The input objects can be supplied using URL parameters, POST parameters or by supplying "Content-Type: application/json" and a JSON blob in the body. There are examples of these below using curl.

+

The response will be a JSON blob in the body of the response. This is formatted to be reasonably human readable.

+

If an error occurs then there will be an HTTP error status (usually 400) and the body of the response will contain a JSON encoded error object.

+

Using POST with URL parameters only

+
curl -X POST 'http://localhost:5572/rc/noop/?potato=1&sausage=2'
+

Response

+
{
+    "potato": "1",
+    "sausage": "2"
+}
+

Here is what an error response looks like:

+
curl -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2'
+
{
+    "error": "arbitrary error on input map[potato:1 sausage:2]",
+    "input": {
+        "potato": "1",
+        "sausage": "2"
+    }
+}
+

Note that curl doesn't return errors to the shell unless you use the -f option

+
$ curl -f -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2'
+curl: (22) The requested URL returned error: 400 Bad Request
+$ echo $?
+22
+

Using POST with a form

+
curl --data "potato=1" --data "sausage=2" http://localhost:5572/rc/noop/
+

Response

+
{
+    "potato": "1",
+    "sausage": "2"
+}
+

Note that you can combine these with URL parameters too with the POST parameters taking precedence.

+
curl --data "potato=1" --data "sausage=2" "http://localhost:5572/rc/noop/?rutabaga=3&sausage=4"
+

Response

+
{
+    "potato": "1",
+    "rutabaga": "3",
+    "sausage": "4"
+}
+
+

Using POST with a JSON blob

+
curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' http://localhost:5572/rc/noop/
+

response

+
{
+    "password": "xyz",
+    "username": "xyz"
+}
+

This can be combined with URL parameters too if required. The JSON blob takes precedence.

+
curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' 'http://localhost:5572/rc/noop/?rutabaga=3&potato=4'
+
{
+    "potato": 2,
+    "rutabaga": "3",
+    "sausage": 1
+}
+

Debugging rclone with pprof

+

If you use the --rc flag this will also enable the use of the go profiling tools on the same port.

+

To use these, first install go.

+

Then (for example) to profile rclone's memory use you can run:

+
go tool pprof -web http://localhost:5572/debug/pprof/heap
+

This should open a page in your browser showing what is using what memory.

+

You can also use the -text flag to produce a textual summary

+
$ go tool pprof -text http://localhost:5572/debug/pprof/heap
+Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
+      flat  flat%   sum%        cum   cum%
+ 1024.03kB 66.62% 66.62%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.addDecoderNode
+     513kB 33.38%   100%      513kB 33.38%  net/http.newBufioWriterSize
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/all.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/serve.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/serve/restic.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init.0
+         0     0%   100%  1024.03kB 66.62%  main.init
+         0     0%   100%      513kB 33.38%  net/http.(*conn).readRequest
+         0     0%   100%      513kB 33.38%  net/http.(*conn).serve
+         0     0%   100%  1024.03kB 66.62%  runtime.main
+

Possible profiles to look at:

+ +

See the net/http/pprof docs for more info on how to use the profiling and for a general overview see the Go team's blog post on profiling go programs.

+

The profiling hook is zero overhead unless it is used.

+

Overview of cloud storage systems

+

Each cloud storage system is slightly different. Rclone attempts to provide a unified interface to them, but some underlying differences show through.

+

Features

+

Here is an overview of the major features of each cloud storage system.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameHashModTimeCase InsensitiveDuplicate FilesMIME Type
Amazon DriveMD5NoYesNoR
Amazon S3MD5YesNoNoR/W
Backblaze B2SHA1YesNoNoR/W
BoxSHA1YesYesNo-
DropboxDBHASH †YesYesNo-
FTP-NoNoNo-
Google Cloud StorageMD5YesNoNoR/W
Google DriveMD5YesNoYesR/W
HTTP-NoNoNoR
HubicMD5YesNoNoR/W
JottacloudMD5YesYesNoR/W
Mega-NoNoYes-
Microsoft Azure Blob StorageMD5YesNoNoR/W
Microsoft OneDriveSHA1 ‡‡YesYesNoR
OpenDriveMD5YesYesNo-
Openstack SwiftMD5YesNoNoR/W
pCloudMD5, SHA1YesNoNoW
QingStorMD5NoNoNoR/W
SFTPMD5, SHA1 ‡YesDependsNo-
WebDAV-Yes ††DependsNo-
Yandex DiskMD5YesNoNoR/W
The local filesystemAllYesDependsNo-
+

Hash

+

The cloud storage system supports various hash types of the objects. The hashes are used when transferring data as an integrity check and can be specifically used with the --checksum flag in syncs and in the check command.

+

To use the verify checksums when transferring between cloud storage systems they must support a common hash type.

+

† Note that Dropbox supports its own custom hash. This is an SHA256 sum of all the 4MB block SHA256s.

+

‡ SFTP supports checksums if the same login has shell access and md5sum or sha1sum as well as echo are in the remote's PATH.

+

†† WebDAV supports modtimes when used with Owncloud and Nextcloud only.

+

‡‡ Microsoft OneDrive Personal supports SHA1 hashes, whereas OneDrive for business and SharePoint server support Microsoft's own QuickXorHash.

+

ModTime

+

The cloud storage system supports setting modification times on objects. If it does then this enables a using the modification times as part of the sync. If not then only the size will be checked by default, though the MD5SUM can be checked with the --checksum flag.

+

All cloud storage systems support some kind of date on the object and these will be set when transferring from the cloud storage system.

+

Case Insensitive

+

If a cloud storage systems is case sensitive then it is possible to have two files which differ only in case, eg file.txt and FILE.txt. If a cloud storage system is case insensitive then that isn't possible.

+

This can cause problems when syncing between a case insensitive system and a case sensitive system. The symptom of this is that no matter how many times you run the sync it never completes fully.

+

The local filesystem and SFTP may or may not be case sensitive depending on OS.

+ +

Most of the time this doesn't cause any problems as people tend to avoid files whose name differs only by case even on case sensitive systems.

+

Duplicate files

+

If a cloud storage system allows duplicate files then it can have two objects with the same name.

+

This confuses rclone greatly when syncing - use the rclone dedupe command to rename or remove duplicates.

+

MIME Type

+

MIME types (also known as media types) classify types of documents using a simple text classification, eg text/html or application/pdf.

+

Some cloud storage systems support reading (R) the MIME type of objects and some support writing (W) the MIME type of objects.

+

The MIME type can be important if you are serving files directly to HTTP from the storage system.

+

If you are copying from a remote which supports reading (R) to a remote which supports writing (W) then rclone will preserve the MIME types. Otherwise they will be guessed from the extension, or the remote itself may assign the MIME type.

+

Optional Features

+

All the remotes support a basic set of features, but there are some optional features supported by some remotes used to make some operations more efficient.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NamePurgeCopyMoveDirMoveCleanUpListRStreamUploadLinkSharingAbout
Amazon DriveYesNoYesYesNo #575NoNoNo #2178No
Amazon S3NoYesNoNoNoYesYesNo #2178No
Backblaze B2NoNoNoNoYesYesYesNo #2178No
BoxYesYesYesYesNo #575NoYesNo #2178No
DropboxYesYesYesYesNo #575NoYesYesYes
FTPNoNoYesYesNoNoYesNo #2178No
Google Cloud StorageYesYesNoNoNoYesYesNo #2178No
Google DriveYesYesYesYesYesYesYesYesYes
HTTPNoNoNoNoNoNoNoNo #2178No
HubicYes †YesNoNoNoYesYesNo #2178Yes
JottacloudYesYesYesYesNoNoNoNoNo
MegaYesNoYesYesNoNoNoNo #2178Yes
Microsoft Azure Blob StorageYesYesNoNoNoYesNoNo #2178No
Microsoft OneDriveYesYesYesYesNo #575NoNoNo #2178Yes
OpenDriveYesYesYesYesNoNoNoNoNo
Openstack SwiftYes †YesNoNoNoYesYesNo #2178Yes
pCloudYesYesYesYesYesNoNoNo #2178Yes
QingStorNoYesNoNoNoYesNoNo #2178No
SFTPNoNoYesYesNoNoYesNo #2178No
WebDAVYesYesYesYesNoNoYes ‡No #2178No
Yandex DiskYesNoNoNoYesYesYesNo #2178No
The local filesystemYesNoYesYesNoNoYesNoYes
+

Purge

+

This deletes a directory quicker than just deleting all the files in the directory.

+

† Note Swift and Hubic implement this in order to delete directory markers but they don't actually have a quicker way of deleting files other than deleting them individually.

+

‡ StreamUpload is not supported with Nextcloud

+

Copy

+

Used when copying an object to and from the same remote. This known as a server side copy so you can copy a file without downloading it and uploading it again. It is used if you use rclone copy or rclone move if the remote doesn't support Move directly.

+

If the server doesn't support Copy directly then for copy operations the file is downloaded then re-uploaded.

+

Move

+

Used when moving/renaming an object on the same remote. This is known as a server side move of a file. This is used in rclone move if the server doesn't support DirMove.

+

If the server isn't capable of Move then rclone simulates it with Copy then delete. If the server doesn't support Copy then rclone will download the file and re-upload it.

+

DirMove

+

This is used to implement rclone move to move a directory if possible. If it isn't then it will use Move on each file (which falls back to Copy then download and upload - see Move section).

+

CleanUp

+

This is used for emptying the trash for a remote by rclone cleanup.

+

If the server can't do CleanUp then rclone cleanup will return an error.

+

ListR

+

The remote supports a recursive list to list all the contents beneath a directory quickly. This enables the --fast-list flag to work. See the rclone docs for more details.

+

StreamUpload

+

Some remotes allow files to be uploaded without knowing the file size in advance. This allows certain operations to work without spooling the file to local disk first, e.g. rclone rcat.

+

LinkSharing

+

Sets the necessary permissions on a file or folder and prints a link that allows others to access them, even if they don't have an account on the particular cloud provider.

+

About

+

This is used to fetch quota information from the remote, like bytes used/free/quota and bytes used in the trash.

+

If the server can't do About then rclone about will return an error.

+

Alias

+

The alias remote provides a new name for another remote.

+

Paths may be as deep as required or a local path, eg remote:directory/subdirectory or /directory/subdirectory.

+

During the initial setup with rclone config you will specify the target remote. The target remote can either be a local path or another remote.

+

Subfolders can be used in target remote. Asume a alias remote named backup with the target mydrive:private/backup. Invoking rclone mkdir backup:desktop is exactly the same as invoking rclone mkdir mydrive:private/backup/desktop.

+

There will be no special handling of paths containing .. segments. Invoking rclone mkdir backup:../desktop is exactly the same as invoking rclone mkdir mydrive:private/backup/../desktop. The empty path is not allowed as a remote. To alias the current directory use . instead.

+

Here is an example of how to make a alias called remote for local folder. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Alias for a existing remote
+   \ "alias"
+ 2 / Amazon Drive
+   \ "amazon cloud drive"
+ 3 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 4 / Backblaze B2
+   \ "b2"
+ 5 / Box
+   \ "box"
+ 6 / Cache a remote
+   \ "cache"
+ 7 / Dropbox
+   \ "dropbox"
+ 8 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 9 / FTP Connection
+   \ "ftp"
+10 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+11 / Google Drive
+   \ "drive"
+12 / Hubic
+   \ "hubic"
+13 / Local Disk
+   \ "local"
+14 / Microsoft Azure Blob Storage
+   \ "azureblob"
+15 / Microsoft OneDrive
+   \ "onedrive"
+16 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+17 / Pcloud
+   \ "pcloud"
+18 / QingCloud Object Storage
+   \ "qingstor"
+19 / SSH/SFTP Connection
+   \ "sftp"
+20 / Webdav
+   \ "webdav"
+21 / Yandex Disk
+   \ "yandex"
+22 / http Connection
+   \ "http"
+Storage> 1
+Remote or path to alias.
+Can be "myremote:path/to/dir", "myremote:bucket", "myremote:" or "/local/path".
+remote> /mnt/storage/backup
+Remote config
+--------------------
+[remote]
+remote = /mnt/storage/backup
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+Current remotes:
+
+Name                 Type
+====                 ====
+remote               alias
+
+e) Edit existing remote
+n) New remote
+d) Delete remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+e/n/d/r/c/s/q> q
+

Once configured you can then use rclone like this,

+

List directories in top level in /mnt/storage/backup

+
rclone lsd remote:
+

List all the files in /mnt/storage/backup

+
rclone ls remote:
+

Copy another local directory to the alias directory called source

+
rclone copy /home/source remote:source
+

Amazon Drive

+

Amazon Drive, formerly known as Amazon Cloud Drive, is a cloud storage service run by Amazon for consumers.

+

Status

+

Important: rclone supports Amazon Drive only if you have your own set of API keys. Unfortunately the Amazon Drive developer program is now closed to new entries so if you don't already have your own set of keys you will not be able to use rclone with Amazon Drive.

+

For the history on why rclone no longer has a set of Amazon Drive API keys see the forum.

+

If you happen to know anyone who works at Amazon then please ask them to re-instate rclone into the Amazon Drive developer program - thanks!

+

Setup

+

The initial setup for Amazon Drive involves getting a token from Amazon which you need to do in your browser. rclone config walks you through it.

+

The configuration process for Amazon Drive may involve using an oauth proxy. This is used to keep the Amazon credentials out of the source code. The proxy runs in Google's very secure App Engine environment and doesn't store any credentials which pass through it.

+

Since rclone doesn't currently have its own Amazon Drive credentials so you will either need to have your own client_id and client_secret with Amazon Drive, or use a a third party ouath proxy in which case you will need to enter client_id, client_secret, auth_url and token_url.

+

Note also if you are not using Amazon's auth_url and token_url, (ie you filled in something for those) then if setting up on a remote machine you can only use the copying the config method of configuration - rclone authorize will not work.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+n/r/c/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / FTP Connection
+   \ "ftp"
+ 7 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 8 / Google Drive
+   \ "drive"
+ 9 / Hubic
+   \ "hubic"
+10 / Local Disk
+   \ "local"
+11 / Microsoft OneDrive
+   \ "onedrive"
+12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+13 / SSH/SFTP Connection
+   \ "sftp"
+14 / Yandex Disk
+   \ "yandex"
+Storage> 1
+Amazon Application Client Id - required.
+client_id> your client ID goes here
+Amazon Application Client Secret - required.
+client_secret> your client secret goes here
+Auth server URL - leave blank to use Amazon's.
+auth_url> Optional auth URL
+Token server url - leave blank to use Amazon's.
+token_url> Optional token URL
+Remote config
+Make sure your Redirect URL is set to "http://127.0.0.1:53682/" in your custom config.
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine
+y) Yes
+n) No
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+client_id = your client ID goes here
+client_secret = your client secret goes here
+auth_url = Optional auth URL
+token_url = Optional token URL
+token = {"access_token":"xxxxxxxxxxxxxxxxxxxxxxx","token_type":"bearer","refresh_token":"xxxxxxxxxxxxxxxxxx","expiry":"2015-09-06T16:07:39.658438471+01:00"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

See the remote setup docs for how to set it up on a machine with no Internet browser available.

+

Note that rclone runs a webserver on your local machine to collect the token as returned from Amazon. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall.

+

Once configured you can then use rclone like this,

+

List directories in top level of your Amazon Drive

+
rclone lsd remote:
+

List all the files in your Amazon Drive

+
rclone ls remote:
+

To copy a local directory to an Amazon Drive directory called backup

+
rclone copy /home/source remote:backup
+

Modified time and MD5SUMs

+

Amazon Drive doesn't allow modification times to be changed via the API so these won't be accurate or used for syncing.

+

It does store MD5SUMs so for a more accurate sync, you can use the --checksum flag.

+

Deleting files

+

Any files you delete with rclone will end up in the trash. Amazon don't provide an API to permanently delete files, nor to empty the trash, so you will have to do that with one of Amazon's apps or via the Amazon Drive website. As of November 17, 2016, files are automatically deleted by Amazon from the trash after 30 days.

+

Using with non .com Amazon accounts

+

Let's say you usually use amazon.co.uk. When you authenticate with rclone it will take you to an amazon.com page to log in. Your amazon.co.uk email and password should work here just fine.

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+ +

Files this size or more will be downloaded via their tempLink. This is to work around a problem with Amazon Drive which blocks downloads of files bigger than about 10GB. The default for this is 9GB which shouldn't need to be changed.

+

To download files above this threshold, rclone requests a tempLink which downloads the file through a temporary URL directly from the underlying S3 storage.

+

--acd-upload-wait-per-gb=TIME

+

Sometimes Amazon Drive gives an error when a file has been fully uploaded but the file appears anyway after a little while. This happens sometimes for files over 1GB in size and nearly every time for files bigger than 10GB. This parameter controls the time rclone waits for the file to appear.

+

The default value for this parameter is 3 minutes per GB, so by default it will wait 3 minutes for every GB uploaded to see if the file appears.

+

You can disable this feature by setting it to 0. This may cause conflict errors as rclone retries the failed upload but the file will most likely appear correctly eventually.

+

These values were determined empirically by observing lots of uploads of big files for a range of file sizes.

+

Upload with the -v flag to see more info about what rclone is doing in this situation.

+

Limitations

+

Note that Amazon Drive is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

+

Amazon Drive has rate limiting so you may notice errors in the sync (429 errors). rclone will automatically retry the sync up to 3 times by default (see --retries flag) which should hopefully work around this problem.

+

Amazon Drive has an internal limit of file sizes that can be uploaded to the service. This limit is not officially published, but all files larger than this will fail.

+

At the time of writing (Jan 2016) is in the area of 50GB per file. This means that larger files are likely to fail.

+

Unfortunately there is no way for rclone to see that this failure is because of file size, so it will retry the operation, as any other failure. To avoid this problem, use --max-size 50000M option to limit the maximum size of uploaded files. Note that --max-size does not split files into segments, it only ignores files over this size.

+

Amazon S3 Storage Providers

+

The S3 backend can be used with a number of different providers:

+ +

Paths are specified as remote:bucket (or remote: for the lsd command.) You may put subdirectories in too, eg remote:bucket/path/to/dir.

+

Once you have made a remote (see the provider specific section above) you can use it like this:

+

See all buckets

+
rclone lsd remote:
+

Make a new bucket

+
rclone mkdir remote:bucket
+

List the contents of a bucket

+
rclone ls remote:bucket
+

Sync /home/local/directory to the remote bucket, deleting any excess files in the bucket.

+
rclone sync /home/local/directory remote:bucket
+

AWS S3

+

Here is an example of making an s3 configuration. First run

+
rclone config
+

This will guide you through an interactive setup process.

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Alias for a existing remote
+   \ "alias"
+ 2 / Amazon Drive
+   \ "amazon cloud drive"
+ 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)
+   \ "s3"
+ 4 / Backblaze B2
+   \ "b2"
+[snip]
+23 / http Connection
+   \ "http"
+Storage> s3
+Choose your S3 provider.
+Choose a number from below, or type in your own value
+ 1 / Amazon Web Services (AWS) S3
+   \ "AWS"
+ 2 / Ceph Object Storage
+   \ "Ceph"
+ 3 / Digital Ocean Spaces
+   \ "DigitalOcean"
+ 4 / Dreamhost DreamObjects
+   \ "Dreamhost"
+ 5 / IBM COS S3
+   \ "IBMCOS"
+ 6 / Minio Object Storage
+   \ "Minio"
+ 7 / Wasabi Object Storage
+   \ "Wasabi"
+ 8 / Any other S3 compatible provider
+   \ "Other"
+provider> 1
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+AWS Access Key ID - leave blank for anonymous access or runtime credentials.
+access_key_id> XXX
+AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
+secret_access_key> YYY
+Region to connect to.
+Choose a number from below, or type in your own value
+   / The default endpoint - a good choice if you are unsure.
+ 1 | US Region, Northern Virginia or Pacific Northwest.
+   | Leave location constraint empty.
+   \ "us-east-1"
+   / US East (Ohio) Region
+ 2 | Needs location constraint us-east-2.
+   \ "us-east-2"
+   / US West (Oregon) Region
+ 3 | Needs location constraint us-west-2.
+   \ "us-west-2"
+   / US West (Northern California) Region
+ 4 | Needs location constraint us-west-1.
+   \ "us-west-1"
+   / Canada (Central) Region
+ 5 | Needs location constraint ca-central-1.
+   \ "ca-central-1"
+   / EU (Ireland) Region
+ 6 | Needs location constraint EU or eu-west-1.
+   \ "eu-west-1"
+   / EU (London) Region
+ 7 | Needs location constraint eu-west-2.
+   \ "eu-west-2"
+   / EU (Frankfurt) Region
+ 8 | Needs location constraint eu-central-1.
+   \ "eu-central-1"
+   / Asia Pacific (Singapore) Region
+ 9 | Needs location constraint ap-southeast-1.
+   \ "ap-southeast-1"
+   / Asia Pacific (Sydney) Region
+10 | Needs location constraint ap-southeast-2.
+   \ "ap-southeast-2"
+   / Asia Pacific (Tokyo) Region
+11 | Needs location constraint ap-northeast-1.
+   \ "ap-northeast-1"
+   / Asia Pacific (Seoul)
+12 | Needs location constraint ap-northeast-2.
+   \ "ap-northeast-2"
+   / Asia Pacific (Mumbai)
+13 | Needs location constraint ap-south-1.
+   \ "ap-south-1"
+   / South America (Sao Paulo) Region
+14 | Needs location constraint sa-east-1.
+   \ "sa-east-1"
+region> 1
+Endpoint for S3 API.
+Leave blank if using AWS to use the default endpoint for the region.
+endpoint> 
+Location constraint - must be set to match the Region. Used when creating buckets only.
+Choose a number from below, or type in your own value
+ 1 / Empty for US Region, Northern Virginia or Pacific Northwest.
+   \ ""
+ 2 / US East (Ohio) Region.
+   \ "us-east-2"
+ 3 / US West (Oregon) Region.
+   \ "us-west-2"
+ 4 / US West (Northern California) Region.
+   \ "us-west-1"
+ 5 / Canada (Central) Region.
+   \ "ca-central-1"
+ 6 / EU (Ireland) Region.
+   \ "eu-west-1"
+ 7 / EU (London) Region.
+   \ "eu-west-2"
+ 8 / EU Region.
+   \ "EU"
+ 9 / Asia Pacific (Singapore) Region.
+   \ "ap-southeast-1"
+10 / Asia Pacific (Sydney) Region.
+   \ "ap-southeast-2"
+11 / Asia Pacific (Tokyo) Region.
+   \ "ap-northeast-1"
+12 / Asia Pacific (Seoul)
+   \ "ap-northeast-2"
+13 / Asia Pacific (Mumbai)
+   \ "ap-south-1"
+14 / South America (Sao Paulo) Region.
+   \ "sa-east-1"
+location_constraint> 1
+Canned ACL used when creating buckets and/or storing objects in S3.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Choose a number from below, or type in your own value
+ 1 / Owner gets FULL_CONTROL. No one else has access rights (default).
+   \ "private"
+ 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access.
+   \ "public-read"
+   / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.
+ 3 | Granting this on a bucket is generally not recommended.
+   \ "public-read-write"
+ 4 / Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access.
+   \ "authenticated-read"
+   / Object owner gets FULL_CONTROL. Bucket owner gets READ access.
+ 5 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
+   \ "bucket-owner-read"
+   / Both the object owner and the bucket owner get FULL_CONTROL over the object.
+ 6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
+   \ "bucket-owner-full-control"
+acl> 1
+The server-side encryption algorithm used when storing this object in S3.
+Choose a number from below, or type in your own value
+ 1 / None
+   \ ""
+ 2 / AES256
+   \ "AES256"
+server_side_encryption> 1
+The storage class to use when storing objects in S3.
+Choose a number from below, or type in your own value
+ 1 / Default
+   \ ""
+ 2 / Standard storage class
+   \ "STANDARD"
+ 3 / Reduced redundancy storage class
+   \ "REDUCED_REDUNDANCY"
+ 4 / Standard Infrequent Access storage class
+   \ "STANDARD_IA"
+ 5 / One Zone Infrequent Access storage class
+   \ "ONEZONE_IA"
+storage_class> 1
+Remote config
+--------------------
+[remote]
+type = s3
+provider = AWS
+env_auth = false
+access_key_id = XXX
+secret_access_key = YYY
+region = us-east-1
+endpoint = 
+location_constraint = 
+acl = private
+server_side_encryption = 
+storage_class = 
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d>
+

--fast-list

+

This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.

+

--update and --use-server-modtime

+

As noted below, the modified time is stored on metadata on the object. It is used by default for all operations that require checking the time a file was last updated. It allows rclone to treat the remote more like a true filesystem, but it is inefficient because it requires an extra API call to retrieve the metadata.

+

For many operations, the time the object was last uploaded to the remote is sufficient to determine if it is "dirty". By using --update along with --use-server-modtime, you can avoid the extra API call and simply upload files whose local modtime is newer than the time it was last uploaded.

+

Modified time

+

The modified time is stored as metadata on the object as X-Amz-Meta-Mtime as floating point since the epoch accurate to 1 ns.

+

Multipart uploads

+

rclone supports multipart uploads with S3 which means that it can upload files bigger than 5GB. Note that files uploaded both with multipart upload and through crypt remotes do not have MD5 sums.

+

Buckets and Regions

+

With Amazon S3 you can list buckets (rclone lsd) using any region, but you can only access the content of a bucket from the region it was created in. If you attempt to access a bucket from the wrong region, you will get an error, incorrect region, the bucket is not in 'XXX' region.

+

Authentication

+

There are a number of ways to supply rclone with a set of AWS credentials, with and without using the environment.

+

The different authentication methods are tried in this order:

+ +

If none of these option actually end up providing rclone with AWS credentials then S3 interaction will be non-authenticated (see below).

+

S3 Permissions

+

When using the sync subcommand of rclone the following minimum permissions are required to be available on the bucket being written to:

+ +

Example policy:

+
{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "arn:aws:iam::USER_SID:user/USER_NAME"
+            },
+            "Action": [
+                "s3:ListBucket",
+                "s3:DeleteObject",
+                "s3:GetObject",
+                "s3:PutObject",
+                "s3:PutObjectAcl"
+            ],
+            "Resource": [
+              "arn:aws:s3:::BUCKET_NAME/*",
+              "arn:aws:s3:::BUCKET_NAME"
+            ]
+        }
+    ]
+}
+

Notes on above:

+
    +
  1. This is a policy that can be used when creating bucket. It assumes that USER_NAME has been created.
    2. +
  2. The Resource entry must include both resource ARNs, as one implies the bucket and the other implies the bucket's objects.
    3. +
+

For reference, here's an Ansible script that will generate one or more buckets that will work with rclone sync.

+

Key Management System (KMS)

+

If you are using server side encryption with KMS then you will find you can't transfer small objects. As a work-around you can use the --ignore-checksum flag.

+

A proper fix is being worked on in issue #1824.

+

Glacier

+

You can transition objects to glacier storage using a lifecycle policy. The bucket can still be synced or copied into normally, but if rclone tries to access the data you will see an error like below.

+
2017/09/11 19:07:43 Failed to sync: failed to open source object: Object in GLACIER, restore first: path/to/file
+

In this case you need to restore the object(s) in question before using rclone.

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--s3-acl=STRING

+

Canned ACL used when creating buckets and/or storing objects in S3.

+

For more info visit the canned ACL docs.

+

--s3-storage-class=STRING

+

Storage class to upload new objects with.

+

Available options include:

+ +

--s3-chunk-size=SIZE

+

Any files larger than this will be uploaded in chunks of this size. The default is 5MB. The minimum is 5MB.

+

Note that 2 chunks of this size are buffered in memory per transfer.

+

If you are transferring large files over high speed links and you have enough memory, then increasing this will speed up the transfers.

+

--s3-force-path-style=BOOL

+

If this is true (the default) then rclone will use path style access, if false then rclone will use virtual path style. See the AWS S3 docs for more info.

+

Some providers (eg Aliyun OSS or Netease COS) require this set to false. It can also be set in the config in the advanced section.

+

--s3-upload-concurrency

+

Number of chunks of the same file that are uploaded concurrently. Default is 2.

+

If you are uploading small amount of large file over high speed link and these uploads do not fully utilize your bandwidth, then increasing this may help to speed up the transfers.

+

Anonymous access to public buckets

+

If you want to use rclone to access a public bucket, configure with a blank access_key_id and secret_access_key. Your config should end up looking like this:

+
[anons3]
+type = s3
+provider = AWS
+env_auth = false
+access_key_id = 
+secret_access_key = 
+region = us-east-1
+endpoint = 
+location_constraint = 
+acl = private
+server_side_encryption = 
+storage_class =
+

Then use it as normal with the name of the public bucket, eg

+
rclone lsd anons3:1000genomes
+

You will be able to list and copy data but not upload it.

+

Ceph

+

Ceph is an open source unified, distributed storage system designed for excellent performance, reliability and scalability. It has an S3 compatible object storage interface.

+

To use rclone with Ceph, configure as above but leave the region blank and set the endpoint. You should end up with something like this in your config:

+
[ceph]
+type = s3
+provider = Ceph
+env_auth = false
+access_key_id = XXX
+secret_access_key = YYY
+region =
+endpoint = https://ceph.endpoint.example.com
+location_constraint =
+acl =
+server_side_encryption =
+storage_class =
+

Note also that Ceph sometimes puts / in the passwords it gives users. If you read the secret access key using the command line tools you will get a JSON blob with the / escaped as \/. Make sure you only write / in the secret access key.

+

Eg the dump from Ceph looks something like this (irrelevant keys removed).

+
{
+    "user_id": "xxx",
+    "display_name": "xxxx",
+    "keys": [
+        {
+            "user": "xxx",
+            "access_key": "xxxxxx",
+            "secret_key": "xxxxxx\/xxxx"
+        }
+    ],
+}
+

Because this is a json dump, it is encoding the / as \/, so if you use the secret key as xxxxxx/xxxx it will work fine.

+

Dreamhost

+

Dreamhost DreamObjects is an object storage system based on CEPH.

+

To use rclone with Dreamhost, configure as above but leave the region blank and set the endpoint. You should end up with something like this in your config:

+
[dreamobjects]
+type = s3
+provider = DreamHost
+env_auth = false
+access_key_id = your_access_key
+secret_access_key = your_secret_key
+region =
+endpoint = objects-us-west-1.dream.io
+location_constraint =
+acl = private
+server_side_encryption =
+storage_class =
+

DigitalOcean Spaces

+

Spaces is an S3-interoperable object storage service from cloud provider DigitalOcean.

+

To connect to DigitalOcean Spaces you will need an access key and secret key. These can be retrieved on the "Applications & API" page of the DigitalOcean control panel. They will be needed when promted by rclone config for your access_key_id and secret_access_key.

+

When prompted for a region or location_constraint, press enter to use the default value. The region must be included in the endpoint setting (e.g. nyc3.digitaloceanspaces.com). The defualt values can be used for other settings.

+

Going through the whole process of creating a new remote by running rclone config, each prompt should be answered as shown below:

+
Storage> s3
+env_auth> 1
+access_key_id> YOUR_ACCESS_KEY
+secret_access_key> YOUR_SECRET_KEY
+region>
+endpoint> nyc3.digitaloceanspaces.com
+location_constraint>
+acl>
+storage_class>
+

The resulting configuration file should look like:

+
[spaces]
+type = s3
+provider = DigitalOcean
+env_auth = false
+access_key_id = YOUR_ACCESS_KEY
+secret_access_key = YOUR_SECRET_KEY
+region =
+endpoint = nyc3.digitaloceanspaces.com
+location_constraint =
+acl =
+server_side_encryption =
+storage_class =
+

Once configured, you can create a new Space and begin copying files. For example:

+
rclone mkdir spaces:my-new-space
+rclone copy /path/to/files spaces:my-new-space
+

IBM COS (S3)

+

Information stored with IBM Cloud Object Storage is encrypted and dispersed across multiple geographic locations, and accessed through an implementation of the S3 API. This service makes use of the distributed storage technologies provided by IBM’s Cloud Object Storage System (formerly Cleversafe). For more information visit: (http://www.ibm.com/cloud/object-storage)

+

To configure access to IBM COS S3, follow the steps below:

+
    +

  1. Run rclone config and select n for a new remote.

    +
    2018/02/14 14:13:11 NOTICE: Config file "C:\\Users\\a\\.config\\rclone\\rclone.conf" not found - using defaults
+No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
    2. +

  2. Enter the name for the configuration

    +
    name> <YOUR NAME>
    3. +

  3. Select "s3" storage.

    +
    Choose a number from below, or type in your own value
+1 / Alias for a existing remote
+\ "alias"
+2 / Amazon Drive
+\ "amazon cloud drive"
+3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, Minio, IBM COS)
+\ "s3"
+4 / Backblaze B2
+\ "b2"
+[snip]
+23 / http Connection
+\ "http"
+Storage> 3
    4. +

  4. Select IBM COS as the S3 Storage Provider.

    +
    Choose the S3 provider.
+Choose a number from below, or type in your own value
+ 1 / Choose this option to configure Storage to AWS S3
+   \ "AWS"
+ 2 / Choose this option to configure Storage to Ceph Systems
+ \ "Ceph"
+ 3 /  Choose this option to configure Storage to Dreamhost
+ \ "Dreamhost"
+   4 / Choose this option to the configure Storage to IBM COS S3
+ \ "IBMCOS"
+ 5 / Choose this option to the configure Storage to Minio
+ \ "Minio"
+ Provider>4
    5. +

  5. Enter the Access Key and Secret.

    +
    AWS Access Key ID - leave blank for anonymous access or runtime credentials.
+access_key_id> <>
+AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
+secret_access_key> <>
    6. +

  6. Specify the endpoint for IBM COS. For Public IBM COS, choose from the option below. For On Premise IBM COS, enter an enpoint address.

    +
    Endpoint for IBM COS S3 API.
+Specify if using an IBM COS On Premise.
+Choose a number from below, or type in your own value
+ 1 / US Cross Region Endpoint
+   \ "s3-api.us-geo.objectstorage.softlayer.net"
+ 2 / US Cross Region Dallas Endpoint
+   \ "s3-api.dal.us-geo.objectstorage.softlayer.net"
+ 3 / US Cross Region Washington DC Endpoint
+   \ "s3-api.wdc-us-geo.objectstorage.softlayer.net"
+ 4 / US Cross Region San Jose Endpoint
+   \ "s3-api.sjc-us-geo.objectstorage.softlayer.net"
+ 5 / US Cross Region Private Endpoint
+   \ "s3-api.us-geo.objectstorage.service.networklayer.com"
+ 6 / US Cross Region Dallas Private Endpoint
+   \ "s3-api.dal-us-geo.objectstorage.service.networklayer.com"
+ 7 / US Cross Region Washington DC Private Endpoint
+   \ "s3-api.wdc-us-geo.objectstorage.service.networklayer.com"
+ 8 / US Cross Region San Jose Private Endpoint
+   \ "s3-api.sjc-us-geo.objectstorage.service.networklayer.com"
+ 9 / US Region East Endpoint
+   \ "s3.us-east.objectstorage.softlayer.net"
+10 / US Region East Private Endpoint
+   \ "s3.us-east.objectstorage.service.networklayer.com"
+11 / US Region South Endpoint
+[snip]
+34 / Toronto Single Site Private Endpoint
+   \ "s3.tor01.objectstorage.service.networklayer.com"
+endpoint>1
    7. +

  7. Specify a IBM COS Location Constraint. The location constraint must match endpoint when using IBM Cloud Public. For on-prem COS, do not make a selection from this list, hit enter

    +
     1 / US Cross Region Standard
+   \ "us-standard"
+ 2 / US Cross Region Vault
+   \ "us-vault"
+ 3 / US Cross Region Cold
+   \ "us-cold"
+ 4 / US Cross Region Flex
+   \ "us-flex"
+ 5 / US East Region Standard
+   \ "us-east-standard"
+ 6 / US East Region Vault
+   \ "us-east-vault"
+ 7 / US East Region Cold
+   \ "us-east-cold"
+ 8 / US East Region Flex
+   \ "us-east-flex"
+ 9 / US South Region Standard
+   \ "us-south-standard"
+10 / US South Region Vault
+   \ "us-south-vault"
+[snip]
+32 / Toronto Flex
+   \ "tor01-flex"
+location_constraint>1
    8. +

  8. Specify a canned ACL. IBM Cloud (Strorage) supports "public-read" and "private". IBM Cloud(Infra) supports all the canned ACLs. On-Premise COS supports all the canned ACLs.

    +
    Canned ACL used when creating buckets and/or storing objects in S3.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Choose a number from below, or type in your own value
+  1 / Owner gets FULL_CONTROL. No one else has access rights (default). This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise COS
+  \ "private"
+  2  / Owner gets FULL_CONTROL. The AllUsers group gets READ access. This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise IBM COS
+  \ "public-read"
+  3 / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. This acl is available on IBM Cloud (Infra), On-Premise IBM COS
+  \ "public-read-write"
+  4  / Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS
+  \ "authenticated-read"
+acl> 1
    9. +

  9. Review the displayed configuration and accept to save the "remote" then quit. The config file should look like this

    +
    [xxx]
+type = s3
+Provider = IBMCOS
+access_key_id = xxx
+secret_access_key = yyy
+endpoint = s3-api.us-geo.objectstorage.softlayer.net
+location_constraint = us-standard
+acl = private
    10. +

  10. Execute rclone commands

    +
    1)  Create a bucket.
+    rclone mkdir IBM-COS-XREGION:newbucket
+2)  List available buckets.
+    rclone lsd IBM-COS-XREGION:
+    -1 2017-11-08 21:16:22        -1 test
+    -1 2018-02-14 20:16:39        -1 newbucket
+3)  List contents of a bucket.
+    rclone ls IBM-COS-XREGION:newbucket
+    18685952 test.exe
+4)  Copy a file from local to remote.
+    rclone copy /Users/file.txt IBM-COS-XREGION:newbucket
+5)  Copy a file from remote to local.
+    rclone copy IBM-COS-XREGION:newbucket/file.txt .
+6)  Delete a file on remote.
+    rclone delete IBM-COS-XREGION:newbucket/file.txt
    11. +
+

Minio

+

Minio is an object storage server built for cloud application developers and devops.

+

It is very easy to install and provides an S3 compatible server which can be used by rclone.

+

To use it, install Minio following the instructions here.

+

When it configures itself Minio will print something like this

+
Endpoint:  http://192.168.1.106:9000  http://172.23.0.1:9000
+AccessKey: USWUXHGYZQYFYFFIT3RE
+SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+Region:    us-east-1
+SQS ARNs:  arn:minio:sqs:us-east-1:1:redis arn:minio:sqs:us-east-1:2:redis
+
+Browser Access:
+   http://192.168.1.106:9000  http://172.23.0.1:9000
+
+Command-line Access: https://docs.minio.io/docs/minio-client-quickstart-guide
+   $ mc config host add myminio http://192.168.1.106:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+
+Object API (Amazon S3 compatible):
+   Go:         https://docs.minio.io/docs/golang-client-quickstart-guide
+   Java:       https://docs.minio.io/docs/java-client-quickstart-guide
+   Python:     https://docs.minio.io/docs/python-client-quickstart-guide
+   JavaScript: https://docs.minio.io/docs/javascript-client-quickstart-guide
+   .NET:       https://docs.minio.io/docs/dotnet-client-quickstart-guide
+
+Drive Capacity: 26 GiB Free, 165 GiB Total
+

These details need to go into rclone config like this. Note that it is important to put the region in as stated above.

+
env_auth> 1
+access_key_id> USWUXHGYZQYFYFFIT3RE
+secret_access_key> MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+region> us-east-1
+endpoint> http://192.168.1.106:9000
+location_constraint>
+server_side_encryption>
+

Which makes the config file look like this

+
[minio]
+type = s3
+provider = Minio
+env_auth = false
+access_key_id = USWUXHGYZQYFYFFIT3RE
+secret_access_key = MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+region = us-east-1
+endpoint = http://192.168.1.106:9000
+location_constraint =
+server_side_encryption =
+

So once set up, for example to copy files into a bucket

+
rclone copy /path/to/files minio:bucket
+

Wasabi

+

Wasabi is a cloud-based object storage service for a broad range of applications and use cases. Wasabi is designed for individuals and organizations that require a high-performance, reliable, and secure data storage infrastructure at minimal cost.

+

Wasabi provides an S3 interface which can be configured for use with rclone like this.

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+n/s> n
+name> wasabi
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+[snip]
+Storage> s3
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+AWS Access Key ID - leave blank for anonymous access or runtime credentials.
+access_key_id> YOURACCESSKEY
+AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
+secret_access_key> YOURSECRETACCESSKEY
+Region to connect to.
+Choose a number from below, or type in your own value
+   / The default endpoint - a good choice if you are unsure.
+ 1 | US Region, Northern Virginia or Pacific Northwest.
+   | Leave location constraint empty.
+   \ "us-east-1"
+[snip]
+region> us-east-1
+Endpoint for S3 API.
+Leave blank if using AWS to use the default endpoint for the region.
+Specify if using an S3 clone such as Ceph.
+endpoint> s3.wasabisys.com
+Location constraint - must be set to match the Region. Used when creating buckets only.
+Choose a number from below, or type in your own value
+ 1 / Empty for US Region, Northern Virginia or Pacific Northwest.
+   \ ""
+[snip]
+location_constraint>
+Canned ACL used when creating buckets and/or storing objects in S3.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Choose a number from below, or type in your own value
+ 1 / Owner gets FULL_CONTROL. No one else has access rights (default).
+   \ "private"
+[snip]
+acl>
+The server-side encryption algorithm used when storing this object in S3.
+Choose a number from below, or type in your own value
+ 1 / None
+   \ ""
+ 2 / AES256
+   \ "AES256"
+server_side_encryption>
+The storage class to use when storing objects in S3.
+Choose a number from below, or type in your own value
+ 1 / Default
+   \ ""
+ 2 / Standard storage class
+   \ "STANDARD"
+ 3 / Reduced redundancy storage class
+   \ "REDUCED_REDUNDANCY"
+ 4 / Standard Infrequent Access storage class
+   \ "STANDARD_IA"
+storage_class>
+Remote config
+--------------------
+[wasabi]
+env_auth = false
+access_key_id = YOURACCESSKEY
+secret_access_key = YOURSECRETACCESSKEY
+region = us-east-1
+endpoint = s3.wasabisys.com
+location_constraint =
+acl =
+server_side_encryption =
+storage_class =
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

This will leave the config file looking like this.

+
[wasabi]
+type = s3
+provider = Wasabi
+env_auth = false
+access_key_id = YOURACCESSKEY
+secret_access_key = YOURSECRETACCESSKEY
+region =
+endpoint = s3.wasabisys.com
+location_constraint =
+acl =
+server_side_encryption =
+storage_class =
+

Aliyun OSS / Netease NOS

+

This describes how to set up Aliyun OSS - Netease NOS is the same except for different endpoints.

+

Note this is a pretty standard S3 setup, except for the setting of force_path_style = false in the advanced config.

+
# rclone config
+e/n/d/r/c/s/q> n
+name> oss
+Type of storage to configure.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)
+   \ "s3"
+Storage> s3
+Choose your S3 provider.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 8 / Any other S3 compatible provider
+   \ "Other"
+provider> other
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Enter a boolean value (true or false). Press Enter for the default ("false").
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+access_key_id> xxxxxxxxxxxx
+AWS Secret Access Key (password)
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+secret_access_key> xxxxxxxxxxxxxxxxx
+Region to connect to.
+Leave blank if you are using an S3 clone and you don't have a region.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 1 / Use this if unsure. Will use v4 signatures and an empty region.
+   \ ""
+ 2 / Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.
+   \ "other-v2-signature"
+region> 1
+Endpoint for S3 API.
+Required when using an S3 clone.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+endpoint> oss-cn-shenzhen.aliyuncs.com
+Location constraint - must be set to match the Region.
+Leave blank if not sure. Used when creating buckets only.
+Enter a string value. Press Enter for the default ("").
+location_constraint>
+Canned ACL used when creating buckets and/or storing objects in S3.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 1 / Owner gets FULL_CONTROL. No one else has access rights (default).
+   \ "private"
+acl> 1
+Edit advanced config? (y/n)
+y) Yes
+n) No
+y/n> y
+Chunk size to use for uploading
+Enter a size with suffix k,M,G,T. Press Enter for the default ("5M").
+chunk_size>
+Don't store MD5 checksum with object metadata
+Enter a boolean value (true or false). Press Enter for the default ("false").
+disable_checksum>
+An AWS session token
+Enter a string value. Press Enter for the default ("").
+session_token>
+Concurrency for multipart uploads.
+Enter a signed integer. Press Enter for the default ("2").
+upload_concurrency>
+If true use path style access if false use virtual hosted style.
+Some providers (eg Aliyun OSS or Netease COS) require this.
+Enter a boolean value (true or false). Press Enter for the default ("true").
+force_path_style> false
+Remote config
+--------------------
+[oss]
+type = s3
+provider = Other
+env_auth = false
+access_key_id = xxxxxxxxx
+secret_access_key = xxxxxxxxxxxxx
+endpoint = oss-cn-shenzhen.aliyuncs.com
+acl = private
+force_path_style = false
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

Backblaze B2

+

B2 is Backblaze's cloud storage system.

+

Paths are specified as remote:bucket (or remote: for the lsd command.) You may put subdirectories in too, eg remote:bucket/path/to/dir.

+

Here is an example of making a b2 configuration. First run

+
rclone config
+

This will guide you through an interactive setup process. You will need your account number (a short hex number) and key (a long hex number) which you can get from the b2 control panel.

+
No remotes found - make a new one
+n) New remote
+q) Quit config
+n/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 7 / Google Drive
+   \ "drive"
+ 8 / Hubic
+   \ "hubic"
+ 9 / Local Disk
+   \ "local"
+10 / Microsoft OneDrive
+   \ "onedrive"
+11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+12 / SSH/SFTP Connection
+   \ "sftp"
+13 / Yandex Disk
+   \ "yandex"
+Storage> 3
+Account ID or Application Key ID
+account> 123456789abc
+Application Key
+key> 0123456789abcdef0123456789abcdef0123456789
+Endpoint for the service - leave blank normally.
+endpoint>
+Remote config
+--------------------
+[remote]
+account = 123456789abc
+key = 0123456789abcdef0123456789abcdef0123456789
+endpoint =
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

This remote is called remote and can now be used like this

+

See all buckets

+
rclone lsd remote:
+

Create a new bucket

+
rclone mkdir remote:bucket
+

List the contents of a bucket

+
rclone ls remote:bucket
+

Sync /home/local/directory to the remote bucket, deleting any excess files in the bucket.

+
rclone sync /home/local/directory remote:bucket
+

Application Keys

+

B2 supports multiple Application Keys for different access permission to B2 Buckets.

+

You can use these with rclone too.

+

Follow Backblaze's docs to create an Application Key with the required permission and add the Application Key ID as the account and the Application Key itself as the key.

+

Note that you must put the Application Key ID as the account - you can't use the master Account ID. If you try then B2 will return 401 errors.

+

--fast-list

+

This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.

+

Modified time

+

The modified time is stored as metadata on the object as X-Bz-Info-src_last_modified_millis as milliseconds since 1970-01-01 in the Backblaze standard. Other tools should be able to use this as a modified time.

+

Modified times are used in syncing and are fully supported except in the case of updating a modification time on an existing object. In this case the object will be uploaded again as B2 doesn't have an API method to set the modification time independent of doing an upload.

+

SHA1 checksums

+

The SHA1 checksums of the files are checked on upload and download and will be used in the syncing process.

+

Large files (bigger than the limit in --b2-upload-cutoff) which are uploaded in chunks will store their SHA1 on the object as X-Bz-Info-large_file_sha1 as recommended by Backblaze.

+

For a large file to be uploaded with an SHA1 checksum, the source needs to support SHA1 checksums. The local disk supports SHA1 checksums so large file transfers from local disk will have an SHA1. See the overview for exactly which remotes support SHA1.

+

Sources which don't support SHA1, in particular crypt will upload large files without SHA1 checksums. This may be fixed in the future (see #1767).

+

Files sizes below --b2-upload-cutoff will always have an SHA1 regardless of the source.

+

Transfers

+

Backblaze recommends that you do lots of transfers simultaneously for maximum speed. In tests from my SSD equipped laptop the optimum setting is about --transfers 32 though higher numbers may be used for a slight speed improvement. The optimum number for you may vary depending on your hardware, how big the files are, how much you want to load your computer, etc. The default of --transfers 4 is definitely too low for Backblaze B2 though.

+

Note that uploading big files (bigger than 200 MB by default) will use a 96 MB RAM buffer by default. There can be at most --transfers of these in use at any moment, so this sets the upper limit on the memory used.

+

Versions

+

When rclone uploads a new version of a file it creates a new version of it. Likewise when you delete a file, the old version will be marked hidden and still be available. Conversely, you may opt in to a "hard delete" of files with the --b2-hard-delete flag which would permanently remove the file instead of hiding it.

+

Old versions of files, where available, are visible using the --b2-versions flag.

+

If you wish to remove all the old versions then you can use the rclone cleanup remote:bucket command which will delete all the old versions of files, leaving the current ones intact. You can also supply a path and only old versions under that path will be deleted, eg rclone cleanup remote:bucket/path/to/stuff.

+

When you purge a bucket, the current and the old versions will be deleted then the bucket will be deleted.

+

However delete will cause the current versions of the files to become hidden old versions.

+

Here is a session showing the listing and retrieval of an old version followed by a cleanup of the old versions.

+

Show current version and all the versions with --b2-versions flag.

+
$ rclone -q ls b2:cleanup-test
+        9 one.txt
+
+$ rclone -q --b2-versions ls b2:cleanup-test
+        9 one.txt
+        8 one-v2016-07-04-141032-000.txt
+       16 one-v2016-07-04-141003-000.txt
+       15 one-v2016-07-02-155621-000.txt
+

Retrieve an old version

+
$ rclone -q --b2-versions copy b2:cleanup-test/one-v2016-07-04-141003-000.txt /tmp
+
+$ ls -l /tmp/one-v2016-07-04-141003-000.txt
+-rw-rw-r-- 1 ncw ncw 16 Jul  2 17:46 /tmp/one-v2016-07-04-141003-000.txt
+

Clean up all the old versions and show that they've gone.

+
$ rclone -q cleanup b2:cleanup-test
+
+$ rclone -q ls b2:cleanup-test
+        9 one.txt
+
+$ rclone -q --b2-versions ls b2:cleanup-test
+        9 one.txt
+

Data usage

+

It is useful to know how many requests are sent to the server in different scenarios.

+

All copy commands send the following 4 requests:

+
/b2api/v1/b2_authorize_account
+/b2api/v1/b2_create_bucket
+/b2api/v1/b2_list_buckets
+/b2api/v1/b2_list_file_names
+

The b2_list_file_names request will be sent once for every 1k files in the remote path, providing the checksum and modification time of the listed files. As of version 1.33 issue #818 causes extra requests to be sent when using B2 with Crypt. When a copy operation does not require any files to be uploaded, no more requests will be sent.

+

Uploading files that do not require chunking, will send 2 requests per file upload:

+
/b2api/v1/b2_get_upload_url
+/b2api/v1/b2_upload_file/
+

Uploading files requiring chunking, will send 2 requests (one each to start and finish the upload) and another 2 requests for each chunk:

+
/b2api/v1/b2_start_large_file
+/b2api/v1/b2_get_upload_part_url
+/b2api/v1/b2_upload_part/
+/b2api/v1/b2_finish_large_file
+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--b2-chunk-size valuee=SIZE

+

When uploading large files chunk the file into this size. Note that these chunks are buffered in memory and there might a maximum of --transfers chunks in progress at once. 5,000,000 Bytes is the minimim size (default 96M).

+

--b2-upload-cutoff=SIZE

+

Cutoff for switching to chunked upload (default 190.735 MiB == 200 MB). Files above this size will be uploaded in chunks of --b2-chunk-size.

+

This value should be set no larger than 4.657GiB (== 5GB) as this is the largest file size that can be uploaded.

+

--b2-test-mode=FLAG

+

This is for debugging purposes only.

+

Setting FLAG to one of the strings below will cause b2 to return specific errors for debugging purposes.

+ +

These will be set in the X-Bz-Test-Mode header which is documented in the b2 integrations checklist.

+

--b2-versions

+

When set rclone will show and act on older versions of files. For example

+

Listing without --b2-versions

+
$ rclone -q ls b2:cleanup-test
+        9 one.txt
+

And with

+
$ rclone -q --b2-versions ls b2:cleanup-test
+        9 one.txt
+        8 one-v2016-07-04-141032-000.txt
+       16 one-v2016-07-04-141003-000.txt
+       15 one-v2016-07-02-155621-000.txt
+

Showing that the current version is unchanged but older versions can be seen. These have the UTC date that they were uploaded to the server to the nearest millisecond appended to them.

+

Note that when using --b2-versions no file write operations are permitted, so you can't upload files or delete them.

+

Box

+

Paths are specified as remote:path

+

Paths may be as deep as required, eg remote:directory/subdirectory.

+

The initial setup for Box involves getting a token from Box which you need to do in your browser. rclone config walks you through it.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Box
+   \ "box"
+ 5 / Dropbox
+   \ "dropbox"
+ 6 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 7 / FTP Connection
+   \ "ftp"
+ 8 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 9 / Google Drive
+   \ "drive"
+10 / Hubic
+   \ "hubic"
+11 / Local Disk
+   \ "local"
+12 / Microsoft OneDrive
+   \ "onedrive"
+13 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+14 / SSH/SFTP Connection
+   \ "sftp"
+15 / Yandex Disk
+   \ "yandex"
+16 / http Connection
+   \ "http"
+Storage> box
+Box App Client Id - leave blank normally.
+client_id> 
+Box App Client Secret - leave blank normally.
+client_secret> 
+Remote config
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine
+y) Yes
+n) No
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+client_id = 
+client_secret = 
+token = {"access_token":"XXX","token_type":"bearer","refresh_token":"XXX","expiry":"XXX"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

See the remote setup docs for how to set it up on a machine with no Internet browser available.

+

Note that rclone runs a webserver on your local machine to collect the token as returned from Box. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall.

+

Once configured you can then use rclone like this,

+

List directories in top level of your Box

+
rclone lsd remote:
+

List all the files in your Box

+
rclone ls remote:
+

To copy a local directory to an Box directory called backup

+
rclone copy /home/source remote:backup
+

Invalid refresh token

+

According to the box docs:

+
+

Each refresh_token is valid for one use in 60 days.

+
+

This means that if you

+ +

then rclone will return an error which includes the text Invalid refresh token.

+

To fix this you will need to use oauth2 again to update the refresh token. You can use the methods in the remote setup docs, bearing in mind that if you use the copy the config file method, you should not use that remote on the computer you did the authentication on.

+

Here is how to do it.

+
$ rclone config
+Current remotes:
+
+Name                 Type
+====                 ====
+remote               box
+
+e) Edit existing remote
+n) New remote
+d) Delete remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+e/n/d/r/c/s/q> e
+Choose a number from below, or type in an existing value
+ 1 > remote
+remote> remote
+--------------------
+[remote]
+type = box
+token = {"access_token":"XXX","token_type":"bearer","refresh_token":"XXX","expiry":"2017-07-08T23:40:08.059167677+01:00"}
+--------------------
+Edit remote
+Value "client_id" = ""
+Edit? (y/n)>
+y) Yes
+n) No
+y/n> n
+Value "client_secret" = ""
+Edit? (y/n)>
+y) Yes
+n) No
+y/n> n
+Remote config
+Already have a token - refresh?
+y) Yes
+n) No
+y/n> y
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine
+y) Yes
+n) No
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+type = box
+token = {"access_token":"YYY","token_type":"bearer","refresh_token":"YYY","expiry":"2017-07-23T12:22:29.259137901+01:00"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

Modified time and hashes

+

Box allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not.

+

Box supports SHA1 type hashes, so you can use the --checksum flag.

+

Transfers

+

For files above 50MB rclone will use a chunked transfer. Rclone will upload up to --transfers chunks at the same time (shared among all the multipart uploads). Chunks are buffered in memory and are normally 8MB so increasing --transfers will increase memory use.

+

Deleting files

+

Depending on the enterprise settings for your user, the item will either be actually deleted from Box or moved to the trash.

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--box-upload-cutoff=SIZE

+

Cutoff for switching to chunked upload - must be >= 50MB. The default is 50MB.

+

--box-commit-retries int

+

Max number of times to try committing a multipart file. (default 100)

+

Limitations

+

Note that Box is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

+

Box file names can't have the \ character in. rclone maps this to and from an identical looking unicode equivalent ＼.

+

Box only supports filenames up to 255 characters in length.

+

Cache (BETA)

+

The cache remote wraps another existing remote and stores file structure and its data for long running tasks like rclone mount.

+

To get started you just need to have an existing remote which can be configured with cache.

+

Here is an example of how to make a remote called test-cache. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+n/r/c/s/q> n
+name> test-cache
+Type of storage to configure.
+Choose a number from below, or type in your own value
+...
+ 5 / Cache a remote
+   \ "cache"
+...
+Storage> 5
+Remote to cache.
+Normally should contain a ':' and a path, eg "myremote:path/to/dir",
+"myremote:bucket" or maybe "myremote:" (not recommended).
+remote> local:/test
+Optional: The URL of the Plex server
+plex_url> http://127.0.0.1:32400
+Optional: The username of the Plex user
+plex_username> dummyusername
+Optional: The password of the Plex user
+y) Yes type in my own password
+g) Generate random password
+n) No leave this optional password blank
+y/g/n> y
+Enter the password:
+password:
+Confirm the password:
+password:
+The size of a chunk. Lower value good for slow connections but can affect seamless reading.
+Default: 5M
+Choose a number from below, or type in your own value
+ 1 / 1MB
+   \ "1m"
+ 2 / 5 MB
+   \ "5M"
+ 3 / 10 MB
+   \ "10M"
+chunk_size> 2
+How much time should object info (file size, file hashes etc) be stored in cache. Use a very high value if you don't plan on changing the source FS from outside the cache.
+Accepted units are: "s", "m", "h".
+Default: 5m
+Choose a number from below, or type in your own value
+ 1 / 1 hour
+   \ "1h"
+ 2 / 24 hours
+   \ "24h"
+ 3 / 24 hours
+   \ "48h"
+info_age> 2
+The maximum size of stored chunks. When the storage grows beyond this size, the oldest chunks will be deleted.
+Default: 10G
+Choose a number from below, or type in your own value
+ 1 / 500 MB
+   \ "500M"
+ 2 / 1 GB
+   \ "1G"
+ 3 / 10 GB
+   \ "10G"
+chunk_total_size> 3
+Remote config
+--------------------
+[test-cache]
+remote = local:/test
+plex_url = http://127.0.0.1:32400
+plex_username = dummyusername
+plex_password = *** ENCRYPTED ***
+chunk_size = 5M
+info_age = 48h
+chunk_total_size = 10G
+

You can then use it like this,

+

List directories in top level of your drive

+
rclone lsd test-cache:
+

List all the files in your drive

+
rclone ls test-cache:
+

To start a cached mount

+
rclone mount --allow-other test-cache: /var/tmp/test-cache
+

Write Features

+

Offline uploading

+

In an effort to make writing through cache more reliable, the backend now supports this feature which can be activated by specifying a cache-tmp-upload-path.

+

A files goes through these states when using this feature:

+
    +
  1. An upload is started (usually by copying a file on the cache remote)
    2. +
  2. When the copy to the temporary location is complete the file is part of the cached remote and looks and behaves like any other file (reading included)
    3. +
  3. After cache-tmp-wait-time passes and the file is next in line, rclone move is used to move the file to the cloud provider
    4. +
  4. Reading the file still works during the upload but most modifications on it will be prohibited
    5. +
  5. Once the move is complete the file is unlocked for modifications as it becomes as any other regular file
    6. +
  6. If the file is being read through cache when it's actually deleted from the temporary path then cache will simply swap the source to the cloud provider without interrupting the reading (small blip can happen though)
    7. +
+

Files are uploaded in sequence and only one file is uploaded at a time. Uploads will be stored in a queue and be processed based on the order they were added. The queue and the temporary storage is persistent across restarts and even purges of the cache.

+

Write Support

+

Writes are supported through cache. One caveat is that a mounted cache remote does not add any retry or fallback mechanism to the upload operation. This will depend on the implementation of the wrapped remote. Consider using Offline uploading for reliable writes.

+

One special case is covered with cache-writes which will cache the file data at the same time as the upload when it is enabled making it available from the cache store immediately once the upload is finished.

+

Read Features

+

Multiple connections

+

To counter the high latency between a local PC where rclone is running and cloud providers, the cache remote can split multiple requests to the cloud provider for smaller file chunks and combines them together locally where they can be available almost immediately before the reader usually needs them.

+

This is similar to buffering when media files are played online. Rclone will stay around the current marker but always try its best to stay ahead and prepare the data before.

+

Plex Integration

+

There is a direct integration with Plex which allows cache to detect during reading if the file is in playback or not. This helps cache to adapt how it queries the cloud provider depending on what is needed for.

+

Scans will have a minimum amount of workers (1) while in a confirmed playback cache will deploy the configured number of workers.

+

This integration opens the doorway to additional performance improvements which will be explored in the near future.

+

Note: If Plex options are not configured, cache will function with its configured options without adapting any of its settings.

+

How to enable? Run rclone config and add all the Plex options (endpoint, username and password) in your remote and it will be automatically enabled.

+

Affected settings: - cache-workers: Configured value during confirmed playback or 1 all the other times

+

Known issues

+

Mount and --dir-cache-time

+

--dir-cache-time controls the first layer of directory caching which works at the mount layer. Being an independent caching mechanism from the cache backend, it will manage its own entries based on the configured time.

+

To avoid getting in a scenario where dir cache has obsolete data and cache would have the correct one, try to set --dir-cache-time to a lower time than --cache-info-age. Default values are already configured in this way.

+

Windows support - Experimental

+

There are a couple of issues with Windows mount functionality that still require some investigations. It should be considered as experimental thus far as fixes come in for this OS.

+

Most of the issues seem to be related to the difference between filesystems on Linux flavors and Windows as cache is heavily dependant on them.

+

Any reports or feedback on how cache behaves on this OS is greatly appreciated.

+ +

Risk of throttling

+

Future iterations of the cache backend will make use of the pooling functionality of the cloud provider to synchronize and at the same time make writing through it more tolerant to failures.

+

There are a couple of enhancements in track to add these but in the meantime there is a valid concern that the expiring cache listings can lead to cloud provider throttles or bans due to repeated queries on it for very large mounts.

+

Some recommendations: - don't use a very small interval for entry informations (--cache-info-age) - while writes aren't yet optimised, you can still write through cache which gives you the advantage of adding the file in the cache at the same time if configured to do so.

+

Future enhancements:

+ +

cache and crypt

+

One common scenario is to keep your data encrypted in the cloud provider using the crypt remote. crypt uses a similar technique to wrap around an existing remote and handles this translation in a seamless way.

+

There is an issue with wrapping the remotes in this order: cloud remote -> crypt -> cache

+

During testing, I experienced a lot of bans with the remotes in this order. I suspect it might be related to how crypt opens files on the cloud provider which makes it think we're downloading the full file instead of small chunks. Organizing the remotes in this order yelds better results: cloud remote -> cache -> crypt

+

Cache and Remote Control (--rc)

+

Cache supports the new --rc mode in rclone and can be remote controlled through the following end points: By default, the listener is disabled if you do not add the flag.

+

rc cache/expire

+

Purge a remote from the cache backend. Supports either a directory or a file. It supports both encrypted and unencrypted file names if cache is wrapped by crypt.

+

Params: - remote = path to remote (required) - withData = true/false to delete cached data (chunks) as well (optional, false by default)

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--cache-db-path=PATH

+

Path to where the file structure metadata (DB) is stored locally. The remote name is used as the DB file name.

+

Default: /cache-backend/ Example: /.cache/cache-backend/test-cache

+

--cache-chunk-path=PATH

+

Path to where partial file data (chunks) is stored locally. The remote name is appended to the final path.

+

This config follows the --cache-db-path. If you specify a custom location for --cache-db-path and don't specify one for --cache-chunk-path then --cache-chunk-path will use the same path as --cache-db-path.

+

Default: /cache-backend/ Example: /.cache/cache-backend/test-cache

+

--cache-db-purge

+

Flag to clear all the cached data for this remote before.

+

Default: not set

+

--cache-chunk-size=SIZE

+

The size of a chunk (partial file data). Use lower numbers for slower connections. If the chunk size is changed, any downloaded chunks will be invalid and cache-chunk-path will need to be cleared or unexpected EOF errors will occur.

+

Default: 5M

+

--cache-total-chunk-size=SIZE

+

The total size that the chunks can take up on the local disk. If cache exceeds this value then it will start to the delete the oldest chunks until it goes under this value.

+

Default: 10G

+

--cache-chunk-clean-interval=DURATION

+

How often should cache perform cleanups of the chunk storage. The default value should be ok for most people. If you find that cache goes over cache-total-chunk-size too often then try to lower this value to force it to perform cleanups more often.

+

Default: 1m

+

--cache-info-age=DURATION

+

How long to keep file structure information (directory listings, file size, mod times etc) locally.

+

If all write operations are done through cache then you can safely make this value very large as the cache store will also be updated in real time.

+

Default: 6h

+

--cache-read-retries=RETRIES

+

How many times to retry a read from a cache storage.

+

Since reading from a cache stream is independent from downloading file data, readers can get to a point where there's no more data in the cache. Most of the times this can indicate a connectivity issue if cache isn't able to provide file data anymore.

+

For really slow connections, increase this to a point where the stream is able to provide data but your experience will be very stuttering.

+

Default: 10

+

--cache-workers=WORKERS

+

How many workers should run in parallel to download chunks.

+

Higher values will mean more parallel processing (better CPU needed) and more concurrent requests on the cloud provider. This impacts several aspects like the cloud provider API limits, more stress on the hardware that rclone runs on but it also means that streams will be more fluid and data will be available much more faster to readers.

+

Note: If the optional Plex integration is enabled then this setting will adapt to the type of reading performed and the value specified here will be used as a maximum number of workers to use. Default: 4

+

--cache-chunk-no-memory

+

By default, cache will keep file data during streaming in RAM as well to provide it to readers as fast as possible.

+

This transient data is evicted as soon as it is read and the number of chunks stored doesn't exceed the number of workers. However, depending on other settings like cache-chunk-size and cache-workers this footprint can increase if there are parallel streams too (multiple files being read at the same time).

+

If the hardware permits it, use this feature to provide an overall better performance during streaming but it can also be disabled if RAM is not available on the local machine.

+

Default: not set

+

--cache-rps=NUMBER

+

This setting places a hard limit on the number of requests per second that cache will be doing to the cloud provider remote and try to respect that value by setting waits between reads.

+

If you find that you're getting banned or limited on the cloud provider through cache and know that a smaller number of requests per second will allow you to work with it then you can use this setting for that.

+

A good balance of all the other settings should make this setting useless but it is available to set for more special cases.

+

NOTE: This will limit the number of requests during streams but other API calls to the cloud provider like directory listings will still pass.

+

Default: disabled

+

--cache-writes

+

If you need to read files immediately after you upload them through cache you can enable this flag to have their data stored in the cache store at the same time during upload.

+

Default: not set

+

--cache-tmp-upload-path=PATH

+

This is the path where cache will use as a temporary storage for new files that need to be uploaded to the cloud provider.

+

Specifying a value will enable this feature. Without it, it is completely disabled and files will be uploaded directly to the cloud provider

+

Default: empty

+

--cache-tmp-wait-time=DURATION

+

This is the duration that a file must wait in the temporary location cache-tmp-upload-path before it is selected for upload.

+

Note that only one file is uploaded at a time and it can take longer to start the upload if a queue formed for this purpose.

+

Default: 15m

+

--cache-db-wait-time=DURATION

+

Only one process can have the DB open at any one time, so rclone waits for this duration for the DB to become available before it gives an error.

+

If you set it to 0 then it will wait forever.

+

Default: 1s

+

Crypt

+

The crypt remote encrypts and decrypts another remote.

+

To use it first set up the underlying remote following the config instructions for that remote. You can also use a local pathname instead of a remote which will encrypt and decrypt from that directory which might be useful for encrypting onto a USB stick for example.

+

First check your chosen remote is working - we'll call it remote:path in these docs. Note that anything inside remote:path will be encrypted and anything outside won't. This means that if you are using a bucket based remote (eg S3, B2, swift) then you should probably put the bucket in the remote s3:bucket. If you just use s3: then rclone will make encrypted bucket names too (if using file name encryption) which may or may not be what you want.

+

Now configure crypt using rclone config. We will call this one secret to differentiate it from the remote.

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n   
+name> secret
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 7 / Google Drive
+   \ "drive"
+ 8 / Hubic
+   \ "hubic"
+ 9 / Local Disk
+   \ "local"
+10 / Microsoft OneDrive
+   \ "onedrive"
+11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+12 / SSH/SFTP Connection
+   \ "sftp"
+13 / Yandex Disk
+   \ "yandex"
+Storage> 5
+Remote to encrypt/decrypt.
+Normally should contain a ':' and a path, eg "myremote:path/to/dir",
+"myremote:bucket" or maybe "myremote:" (not recommended).
+remote> remote:path
+How to encrypt the filenames.
+Choose a number from below, or type in your own value
+ 1 / Don't encrypt the file names.  Adds a ".bin" extension only.
+   \ "off"
+ 2 / Encrypt the filenames see the docs for the details.
+   \ "standard"
+ 3 / Very simple filename obfuscation.
+   \ "obfuscate"
+filename_encryption> 2
+Option to either encrypt directory names or leave them intact.
+Choose a number from below, or type in your own value
+ 1 / Encrypt directory names.
+   \ "true"
+ 2 / Don't encrypt directory names, leave them intact.
+   \ "false"
+filename_encryption> 1
+Password or pass phrase for encryption.
+y) Yes type in my own password
+g) Generate random password
+y/g> y
+Enter the password:
+password:
+Confirm the password:
+password:
+Password or pass phrase for salt. Optional but recommended.
+Should be different to the previous password.
+y) Yes type in my own password
+g) Generate random password
+n) No leave this optional password blank
+y/g/n> g
+Password strength in bits.
+64 is just about memorable
+128 is secure
+1024 is the maximum
+Bits> 128
+Your password is: JAsJvRcgR-_veXNfy_sGmQ
+Use this password?
+y) Yes
+n) No
+y/n> y
+Remote config
+--------------------
+[secret]
+remote = remote:path
+filename_encryption = standard
+password = *** ENCRYPTED ***
+password2 = *** ENCRYPTED ***
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

Important The password is stored in the config file is lightly obscured so it isn't immediately obvious what it is. It is in no way secure unless you use config file encryption.

+

A long passphrase is recommended, or you can use a random one. Note that if you reconfigure rclone with the same passwords/passphrases elsewhere it will be compatible - all the secrets used are derived from those two passwords/passphrases.

+

Note that rclone does not encrypt

+ +

Specifying the remote

+

In normal use, make sure the remote has a : in. If you specify the remote without a : then rclone will use a local directory of that name. So if you use a remote of /path/to/secret/files then rclone will encrypt stuff to that directory. If you use a remote of name then rclone will put files in a directory called name in the current directory.

+

If you specify the remote as remote:path/to/dir then rclone will store encrypted files in path/to/dir on the remote. If you are using file name encryption, then when you save files to secret:subdir/subfile this will store them in the unencrypted path path/to/dir but the subdir/subpath bit will be encrypted.

+

Note that unless you want encrypted bucket names (which are difficult to manage because you won't know what directory they represent in web interfaces etc), you should probably specify a bucket, eg remote:secretbucket when using bucket based remotes such as S3, Swift, Hubic, B2, GCS.

+

Example

+

To test I made a little directory of files using "standard" file name encryption.

+
plaintext/
+├── file0.txt
+├── file1.txt
+└── subdir
+    ├── file2.txt
+    ├── file3.txt
+    └── subsubdir
+        └── file4.txt
+

Copy these to the remote and list them back

+
$ rclone -q copy plaintext secret:
+$ rclone -q ls secret:
+        7 file1.txt
+        6 file0.txt
+        8 subdir/file2.txt
+       10 subdir/subsubdir/file4.txt
+        9 subdir/file3.txt
+

Now see what that looked like when encrypted

+
$ rclone -q ls remote:path
+       55 hagjclgavj2mbiqm6u6cnjjqcg
+       54 v05749mltvv1tf4onltun46gls
+       57 86vhrsv86mpbtd3a0akjuqslj8/dlj7fkq4kdq72emafg7a7s41uo
+       58 86vhrsv86mpbtd3a0akjuqslj8/7uu829995du6o42n32otfhjqp4/b9pausrfansjth5ob3jkdqd4lc
+       56 86vhrsv86mpbtd3a0akjuqslj8/8njh1sk437gttmep3p70g81aps
+

Note that this retains the directory structure which means you can do this

+
$ rclone -q ls secret:subdir
+        8 file2.txt
+        9 file3.txt
+       10 subsubdir/file4.txt
+

If don't use file name encryption then the remote will look like this - note the .bin extensions added to prevent the cloud provider attempting to interpret the data.

+
$ rclone -q ls remote:path
+       54 file0.txt.bin
+       57 subdir/file3.txt.bin
+       56 subdir/file2.txt.bin
+       58 subdir/subsubdir/file4.txt.bin
+       55 file1.txt.bin
+

File name encryption modes

+

Here are some of the features of the file name encryption modes

+

Off

+ +

Standard

+ +

Obfuscation

+

This is a simple "rotate" of the filename, with each file having a rot distance based on the filename. We store the distance at the beginning of the filename. So a file called "hello" may become "53.jgnnq"

+

This is not a strong encryption of filenames, but it may stop automated scanning tools from picking up on filename patterns. As such it's an intermediate between "off" and "standard". The advantage is that it allows for longer path segment names.

+

There is a possibility with some unicode based filenames that the obfuscation is weak and may map lower case characters to upper case equivalents. You can not rely on this for strong protection.

+ +

Cloud storage systems have various limits on file name length and total path length which you are more likely to hit using "Standard" file name encryption. If you keep your file names to below 156 characters in length then you should be OK on all providers.

+

There may be an even more secure file name encryption mode in the future which will address the long file name problem.

+

Directory name encryption

+

Crypt offers the option of encrypting dir names or leaving them intact. There are two options:

+

True

+

Encrypts the whole file path including directory names Example: 1/12/123.txt is encrypted to p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0

+

False

+

Only encrypts file names, skips directory names Example: 1/12/123.txt is encrypted to 1/12/qgm4avr35m5loi1th53ato71v0

+

Modified time and hashes

+

Crypt stores modification times using the underlying remote so support depends on that.

+

Hashes are not stored for crypt. However the data integrity is protected by an extremely strong crypto authenticator.

+

Note that you should use the rclone cryptcheck command to check the integrity of a crypted remote instead of rclone check which can't check the checksums properly.

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--crypt-show-mapping

+

If this flag is set then for each file that the remote is asked to list, it will log (at level INFO) a line stating the decrypted file name and the encrypted file name.

+

This is so you can work out which encrypted names are which decrypted names just in case you need to do something with the encrypted file names, or for debugging purposes.

+

Backing up a crypted remote

+

If you wish to backup a crypted remote, it it recommended that you use rclone sync on the encrypted files, and make sure the passwords are the same in the new encrypted remote.

+

This will have the following advantages

+ +

For example, let's say you have your original remote at remote: with the encrypted version at eremote: with path remote:crypt. You would then set up the new remote remote2: and then the encrypted version eremote2: with path remote2:crypt using the same passwords as eremote:.

+

To sync the two remotes you would do

+
rclone sync remote:crypt remote2:crypt
+

And to check the integrity you would do

+
rclone check remote:crypt remote2:crypt
+

File formats

+

File encryption

+

Files are encrypted 1:1 source file to destination object. The file has a header and is divided into chunks.

+ + +

The initial nonce is generated from the operating systems crypto strong random number generator. The nonce is incremented for each chunk read making sure each nonce is unique for each block written. The chance of a nonce being re-used is minuscule. If you wrote an exabyte of data (10¹⁸ bytes) you would have a probability of approximately 2×10⁻³² of re-using a nonce.

+

Chunk

+

Each chunk will contain 64kB of data, except for the last one which may have less data. The data chunk is in standard NACL secretbox format. Secretbox uses XSalsa20 and Poly1305 to encrypt and authenticate messages.

+

Each chunk contains:

+ +

64k chunk size was chosen as the best performing chunk size (the authenticator takes too much time below this and the performance drops off due to cache effects above this). Note that these chunks are buffered in memory so they can't be too big.

+

This uses a 32 byte (256 bit key) key derived from the user password.

+

Examples

+

1 byte file will encrypt to

+ +

49 bytes total

+

1MB (1048576 bytes) file will encrypt to

+ +

1049120 bytes total (a 0.05% overhead). This is the overhead for big files.

+

Name encryption

+

File names are encrypted segment by segment - the path is broken up into / separated strings and these are encrypted individually.

+

File segments are padded using using PKCS#7 to a multiple of 16 bytes before encryption.

+

They are then encrypted with EME using AES with 256 bit key. EME (ECB-Mix-ECB) is a wide-block encryption mode presented in the 2003 paper "A Parallelizable Enciphering Mode" by Halevi and Rogaway.

+

This makes for deterministic encryption which is what we want - the same filename must encrypt to the same thing otherwise we can't find it on the cloud storage system.

+

This means that

+ +

This uses a 32 byte key (256 bits) and a 16 byte (128 bits) IV both of which are derived from the user password.

+

After encryption they are written out using a modified version of standard base32 encoding as described in RFC4648. The standard encoding is modified in two ways:

+ +

base32 is used rather than the more efficient base64 so rclone can be used on case insensitive remotes (eg Windows, Amazon Drive).

+

Key derivation

+

Rclone uses scrypt with parameters N=16384, r=8, p=1 with an optional user supplied salt (password2) to derive the 32+32+16 = 80 bytes of key material required. If the user doesn't supply a salt then rclone uses an internal one.

+

scrypt makes it impractical to mount a dictionary attack on rclone encrypted data. For full protection against this you should always use a salt.

+

Dropbox

+

Paths are specified as remote:path

+

Dropbox paths may be as deep as required, eg remote:directory/subdirectory.

+

The initial setup for dropbox involves getting a token from Dropbox which you need to do in your browser. rclone config walks you through it.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
n) New remote
+d) Delete remote
+q) Quit config
+e/n/d/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 7 / Google Drive
+   \ "drive"
+ 8 / Hubic
+   \ "hubic"
+ 9 / Local Disk
+   \ "local"
+10 / Microsoft OneDrive
+   \ "onedrive"
+11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+12 / SSH/SFTP Connection
+   \ "sftp"
+13 / Yandex Disk
+   \ "yandex"
+Storage> 4
+Dropbox App Key - leave blank normally.
+app_key>
+Dropbox App Secret - leave blank normally.
+app_secret>
+Remote config
+Please visit:
+https://www.dropbox.com/1/oauth2/authorize?client_id=XXXXXXXXXXXXXXX&response_type=code
+Enter the code: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXXXXXXXX
+--------------------
+[remote]
+app_key =
+app_secret =
+token = XXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXX_XXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

You can then use it like this,

+

List directories in top level of your dropbox

+
rclone lsd remote:
+

List all the files in your dropbox

+
rclone ls remote:
+

To copy a local directory to a dropbox directory called backup

+
rclone copy /home/source remote:backup
+

Dropbox for business

+

Rclone supports Dropbox for business and Team Folders.

+

When using Dropbox for business remote: and remote:path/to/file will refer to your personal folder.

+

If you wish to see Team Folders you must use a leading / in the path, so rclone lsd remote:/ will refer to the root and show you all Team Folders and your User Folder.

+

You can then use team folders like this remote:/TeamFolder and remote:/TeamFolder/path/to/file.

+

A leading / for a Dropbox personal account will do nothing, but it will take an extra HTTP transaction so it should be avoided.

+

Modified time and Hashes

+

Dropbox supports modified times, but the only way to set a modification time is to re-upload the file.

+

This means that if you uploaded your data with an older version of rclone which didn't support the v2 API and modified times, rclone will decide to upload all your old data to fix the modification times. If you don't want this to happen use --size-only or --checksum flag to stop it.

+

Dropbox supports its own hash type which is checked for all transfers.

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--dropbox-chunk-size=SIZE

+

Any files larger than this will be uploaded in chunks of this size. The default is 48MB. The maximum is 150MB.

+

Note that chunks are buffered in memory (one at a time) so rclone can deal with retries. Setting this larger will increase the speed slightly (at most 10% for 128MB in tests) at the cost of using more memory. It can be set smaller if you are tight on memory.

+

Limitations

+

Note that Dropbox is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

+

There are some file names such as thumbs.db which Dropbox can't store. There is a full list of them in the "Ignored Files" section of this document. Rclone will issue an error message File name disallowed - not uploading if it attempts to upload one of those file names, but the sync won't fail.

+

If you have more than 10,000 files in a directory then rclone purge dropbox:dir will return the error Failed to purge: There are too many files involved in this operation. As a work-around do an rclone delete dropbox:dir followed by an rclone rmdir dropbox:dir.

+

FTP

+

FTP is the File Transfer Protocol. FTP support is provided using the github.com/jlaffaye/ftp package.

+

Here is an example of making an FTP configuration. First run

+
rclone config
+

This will guide you through an interactive setup process. An FTP remote only needs a host together with and a username and a password. With anonymous FTP server, you will need to use anonymous as username and your email address as the password.

+
No remotes found - make a new one
+n) New remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+n/r/c/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / FTP Connection 
+   \ "ftp"
+ 7 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 8 / Google Drive
+   \ "drive"
+ 9 / Hubic
+   \ "hubic"
+10 / Local Disk
+   \ "local"
+11 / Microsoft OneDrive
+   \ "onedrive"
+12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+13 / SSH/SFTP Connection
+   \ "sftp"
+14 / Yandex Disk
+   \ "yandex"
+Storage> ftp
+FTP host to connect to
+Choose a number from below, or type in your own value
+ 1 / Connect to ftp.example.com
+   \ "ftp.example.com"
+host> ftp.example.com
+FTP username, leave blank for current username, ncw
+user>
+FTP port, leave blank to use default (21)
+port>
+FTP password
+y) Yes type in my own password
+g) Generate random password
+y/g> y
+Enter the password:
+password:
+Confirm the password:
+password:
+Remote config
+--------------------
+[remote]
+host = ftp.example.com
+user = 
+port =
+pass = *** ENCRYPTED ***
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

This remote is called remote and can now be used like this

+

See all directories in the home directory

+
rclone lsd remote:
+

Make a new directory

+
rclone mkdir remote:path/to/directory
+

List the contents of a directory

+
rclone ls remote:path/to/directory
+

Sync /home/local/directory to the remote directory, deleting any excess files in the directory.

+
rclone sync /home/local/directory remote:directory
+

Modified time

+

FTP does not support modified times. Any times you see on the server will be time of upload.

+

Checksums

+

FTP does not support any checksums.

+

Limitations

+

Note that since FTP isn't HTTP based the following flags don't work with it: --dump-headers, --dump-bodies, --dump-auth

+

Note that --timeout isn't supported (but --contimeout is).

+

Note that --bind isn't supported.

+

FTP could support server side move but doesn't yet.

+

Google Cloud Storage

+

Paths are specified as remote:bucket (or remote: for the lsd command.) You may put subdirectories in too, eg remote:bucket/path/to/dir.

+

The initial setup for google cloud storage involves getting a token from Google Cloud Storage which you need to do in your browser. rclone config walks you through it.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
n) New remote
+d) Delete remote
+q) Quit config
+e/n/d/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 7 / Google Drive
+   \ "drive"
+ 8 / Hubic
+   \ "hubic"
+ 9 / Local Disk
+   \ "local"
+10 / Microsoft OneDrive
+   \ "onedrive"
+11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+12 / SSH/SFTP Connection
+   \ "sftp"
+13 / Yandex Disk
+   \ "yandex"
+Storage> 6
+Google Application Client Id - leave blank normally.
+client_id>
+Google Application Client Secret - leave blank normally.
+client_secret>
+Project number optional - needed only for list/create/delete buckets - see your developer console.
+project_number> 12345678
+Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login.
+service_account_file>
+Access Control List for new objects.
+Choose a number from below, or type in your own value
+ 1 / Object owner gets OWNER access, and all Authenticated Users get READER access.
+   \ "authenticatedRead"
+ 2 / Object owner gets OWNER access, and project team owners get OWNER access.
+   \ "bucketOwnerFullControl"
+ 3 / Object owner gets OWNER access, and project team owners get READER access.
+   \ "bucketOwnerRead"
+ 4 / Object owner gets OWNER access [default if left blank].
+   \ "private"
+ 5 / Object owner gets OWNER access, and project team members get access according to their roles.
+   \ "projectPrivate"
+ 6 / Object owner gets OWNER access, and all Users get READER access.
+   \ "publicRead"
+object_acl> 4
+Access Control List for new buckets.
+Choose a number from below, or type in your own value
+ 1 / Project team owners get OWNER access, and all Authenticated Users get READER access.
+   \ "authenticatedRead"
+ 2 / Project team owners get OWNER access [default if left blank].
+   \ "private"
+ 3 / Project team members get access according to their roles.
+   \ "projectPrivate"
+ 4 / Project team owners get OWNER access, and all Users get READER access.
+   \ "publicRead"
+ 5 / Project team owners get OWNER access, and all Users get WRITER access.
+   \ "publicReadWrite"
+bucket_acl> 2
+Location for the newly created buckets.
+Choose a number from below, or type in your own value
+ 1 / Empty for default location (US).
+   \ ""
+ 2 / Multi-regional location for Asia.
+   \ "asia"
+ 3 / Multi-regional location for Europe.
+   \ "eu"
+ 4 / Multi-regional location for United States.
+   \ "us"
+ 5 / Taiwan.
+   \ "asia-east1"
+ 6 / Tokyo.
+   \ "asia-northeast1"
+ 7 / Singapore.
+   \ "asia-southeast1"
+ 8 / Sydney.
+   \ "australia-southeast1"
+ 9 / Belgium.
+   \ "europe-west1"
+10 / London.
+   \ "europe-west2"
+11 / Iowa.
+   \ "us-central1"
+12 / South Carolina.
+   \ "us-east1"
+13 / Northern Virginia.
+   \ "us-east4"
+14 / Oregon.
+   \ "us-west1"
+location> 12
+The storage class to use when storing objects in Google Cloud Storage.
+Choose a number from below, or type in your own value
+ 1 / Default
+   \ ""
+ 2 / Multi-regional storage class
+   \ "MULTI_REGIONAL"
+ 3 / Regional storage class
+   \ "REGIONAL"
+ 4 / Nearline storage class
+   \ "NEARLINE"
+ 5 / Coldline storage class
+   \ "COLDLINE"
+ 6 / Durable reduced availability storage class
+   \ "DURABLE_REDUCED_AVAILABILITY"
+storage_class> 5
+Remote config
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine or Y didn't work
+y) Yes
+n) No
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+type = google cloud storage
+client_id =
+client_secret =
+token = {"AccessToken":"xxxx.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","RefreshToken":"x/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_xxxxxxxxx","Expiry":"2014-07-17T20:49:14.929208288+01:00","Extra":null}
+project_number = 12345678
+object_acl = private
+bucket_acl = private
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

Note that rclone runs a webserver on your local machine to collect the token as returned from Google if you use auto config mode. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall, or use manual mode.

+

This remote is called remote and can now be used like this

+

See all the buckets in your project

+
rclone lsd remote:
+

Make a new bucket

+
rclone mkdir remote:bucket
+

List the contents of a bucket

+
rclone ls remote:bucket
+

Sync /home/local/directory to the remote bucket, deleting any excess files in the bucket.

+
rclone sync /home/local/directory remote:bucket
+

Service Account support

+

You can set up rclone with Google Cloud Storage in an unattended mode, i.e. not tied to a specific end-user Google account. This is useful when you want to synchronise files onto machines that don't have actively logged-in users, for example build machines.

+

To get credentials for Google Cloud Platform IAM Service Accounts, please head to the Service Account section of the Google Developer Console. Service Accounts behave just like normal User permissions in Google Cloud Storage ACLs, so you can limit their access (e.g. make them read only). After creating an account, a JSON file containing the Service Account's credentials will be downloaded onto your machines. These credentials are what rclone will use for authentication.

+

To use a Service Account instead of OAuth2 token flow, enter the path to your Service Account credentials at the service_account_file prompt and rclone won't use the browser based authentication flow. If you'd rather stuff the contents of the credentials file into the rclone config file, you can set service_account_credentials with the actual contents of the file instead, or set the equivalent environment variable.

+

--fast-list

+

This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.

+

Modified time

+

Google google cloud storage stores md5sums natively and rclone stores modification times as metadata on the object, under the "mtime" key in RFC3339 format accurate to 1ns.

+

Google Drive

+

Paths are specified as drive:path

+

Drive paths may be as deep as required, eg drive:directory/subdirectory.

+

The initial setup for drive involves getting a token from Google drive which you need to do in your browser. rclone config walks you through it.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+n/r/c/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+[snip]
+10 / Google Drive
+   \ "drive"
+[snip]
+Storage> drive
+Google Application Client Id - leave blank normally.
+client_id>
+Google Application Client Secret - leave blank normally.
+client_secret>
+Scope that rclone should use when requesting access from drive.
+Choose a number from below, or type in your own value
+ 1 / Full access all files, excluding Application Data Folder.
+   \ "drive"
+ 2 / Read-only access to file metadata and file contents.
+   \ "drive.readonly"
+   / Access to files created by rclone only.
+ 3 | These are visible in the drive website.
+   | File authorization is revoked when the user deauthorizes the app.
+   \ "drive.file"
+   / Allows read and write access to the Application Data folder.
+ 4 | This is not visible in the drive website.
+   \ "drive.appfolder"
+   / Allows read-only access to file metadata but
+ 5 | does not allow any access to read or download file content.
+   \ "drive.metadata.readonly"
+scope> 1
+ID of the root folder - leave blank normally.  Fill in to access "Computers" folders. (see docs).
+root_folder_id> 
+Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login.
+service_account_file>
+Remote config
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine or Y didn't work
+y) Yes
+n) No
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+Configure this as a team drive?
+y) Yes
+n) No
+y/n> n
+--------------------
+[remote]
+client_id = 
+client_secret = 
+scope = drive
+root_folder_id = 
+service_account_file =
+token = {"access_token":"XXX","token_type":"Bearer","refresh_token":"XXX","expiry":"2014-03-16T13:57:58.955387075Z"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

Note that rclone runs a webserver on your local machine to collect the token as returned from Google if you use auto config mode. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall, or use manual mode.

+

You can then use it like this,

+

List directories in top level of your drive

+
rclone lsd remote:
+

List all the files in your drive

+
rclone ls remote:
+

To copy a local directory to a drive directory called backup

+
rclone copy /home/source remote:backup
+

Scopes

+

Rclone allows you to select which scope you would like for rclone to use. This changes what type of token is granted to rclone. The scopes are defined here..

+

The scope are

+

drive

+

This is the default scope and allows full access to all files, except for the Application Data Folder (see below).

+

Choose this one if you aren't sure.

+

drive.readonly

+

This allows read only access to all files. Files may be listed and downloaded but not uploaded, renamed or deleted.

+

drive.file

+

With this scope rclone can read/view/modify only those files and folders it creates.

+

So if you uploaded files to drive via the web interface (or any other means) they will not be visible to rclone.

+

This can be useful if you are using rclone to backup data and you want to be sure confidential data on your drive is not visible to rclone.

+

Files created with this scope are visible in the web interface.

+

drive.appfolder

+

This gives rclone its own private area to store files. Rclone will not be able to see any other files on your drive and you won't be able to see rclone's files from the web interface either.

+

drive.metadata.readonly

+

This allows read only access to file names only. It does not allow rclone to download or upload data, or rename or delete files or directories.

+

Root folder ID

+

You can set the root_folder_id for rclone. This is the directory (identified by its Folder ID) that rclone considers to be a the root of your drive.

+

Normally you will leave this blank and rclone will determine the correct root to use itself.

+

However you can set this to restrict rclone to a specific folder hierarchy or to access data within the "Computers" tab on the drive web interface (where files from Google's Backup and Sync desktop program go).

+

In order to do this you will have to find the Folder ID of the directory you wish rclone to display. This will be the last segment of the URL when you open the relevant folder in the drive web interface.

+

So if the folder you want rclone to use has a URL which looks like https://drive.google.com/drive/folders/1XyfxxxxxxxxxxxxxxxxxxxxxxxxxKHCh in the browser, then you use 1XyfxxxxxxxxxxxxxxxxxxxxxxxxxKHCh as the root_folder_id in the config.

+

NB folders under the "Computers" tab seem to be read only (drive gives a 500 error) when using rclone.

+

There doesn't appear to be an API to discover the folder IDs of the "Computers" tab - please contact us if you know otherwise!

+

Note also that rclone can't access any data under the "Backups" tab on the google drive web interface yet.

+

Service Account support

+

You can set up rclone with Google Drive in an unattended mode, i.e. not tied to a specific end-user Google account. This is useful when you want to synchronise files onto machines that don't have actively logged-in users, for example build machines.

+

To use a Service Account instead of OAuth2 token flow, enter the path to your Service Account credentials at the service_account_file prompt during rclone config and rclone won't use the browser based authentication flow. If you'd rather stuff the contents of the credentials file into the rclone config file, you can set service_account_credentials with the actual contents of the file instead, or set the equivalent environment variable.

+

Use case - Google Apps/G-suite account and individual Drive

+

Let's say that you are the administrator of a Google Apps (old) or G-suite account. The goal is to store data on an individual's Drive account, who IS a member of the domain. We'll call the domain example.com, and the user foo@example.com.

+

There's a few steps we need to go through to accomplish this:

+
1. Create a service account for example.com
+ +
2. Allowing API access to example.com Google Drive
+ +
3. Configure rclone, assuming a new install
+
rclone config
+
+n/s/q> n         # New
+name>gdrive      # Gdrive is an example name
+Storage>         # Select the number shown for Google Drive
+client_id>       # Can be left blank
+client_secret>   # Can be left blank
+scope>           # Select your scope, 1 for example
+root_folder_id>  # Can be left blank
+service_account_file> /home/foo/myJSONfile.json # This is where the JSON file goes!
+y/n>             # Auto config, y
+
+
4. Verify that it's working
+ +

Team drives

+

If you want to configure the remote to point to a Google Team Drive then answer y to the question Configure this as a team drive?.

+

This will fetch the list of Team Drives from google and allow you to configure which one you want to use. You can also type in a team drive ID if you prefer.

+

For example:

+
Configure this as a team drive?
+y) Yes
+n) No
+y/n> y
+Fetching team drive list...
+Choose a number from below, or type in your own value
+ 1 / Rclone Test
+   \ "xxxxxxxxxxxxxxxxxxxx"
+ 2 / Rclone Test 2
+   \ "yyyyyyyyyyyyyyyyyyyy"
+ 3 / Rclone Test 3
+   \ "zzzzzzzzzzzzzzzzzzzz"
+Enter a Team Drive ID> 1
+--------------------
+[remote]
+client_id =
+client_secret =
+token = {"AccessToken":"xxxx.x.xxxxx_xxxxxxxxxxx_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","RefreshToken":"1/xxxxxxxxxxxxxxxx_xxxxxxxxxxxxxxxxxxxxxxxxxx","Expiry":"2014-03-16T13:57:58.955387075Z","Extra":null}
+team_drive = xxxxxxxxxxxxxxxxxxxx
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

--fast-list

+

This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.

+

It does this by combining multiple list calls into a single API request.

+

This works by combining many '%s' in parents filters into one expression. To list the contents of directories a, b and c, the the following requests will be send by the regular List function:

+
trashed=false and 'a' in parents
+trashed=false and 'b' in parents
+trashed=false and 'c' in parents
+

These can now be combined into a single request:

+
trashed=false and ('a' in parents or 'b' in parents or 'c' in parents)
+

The implementation of ListR will put up to 50 parents filters into one request. It will use the --checkers value to specify the number of requests to run in parallel.

+

In tests, these batch requests were up to 20x faster than the regular method. Running the following command against different sized folders gives:

+
rclone lsjson -vv -R --checkers=6 gdrive:folder
+

small folder (220 directories, 700 files):

+ +

large folder (10600 directories, 39000 files):

+ +

Modified time

+

Google drive stores modification times accurate to 1 ms.

+

Revisions

+

Google drive stores revisions of files. When you upload a change to an existing file to google drive using rclone it will create a new revision of that file.

+

Revisions follow the standard google policy which at time of writing was

+ +

Deleting files

+

By default rclone will send all files to the trash when deleting files. If deleting them permanently is required then use the --drive-use-trash=false flag, or set the equivalent environment variable.

+

Emptying trash

+

If you wish to empty your trash you can use the rclone cleanup remote: command which will permanently delete all your trashed files. This command does not take any path arguments.

+

Quota information

+

To view your current quota you can use the rclone about remote: command which will display your usage limit (quota), the usage in Google Drive, the size of all files in the Trash and the space used by other Google services such as Gmail. This command does not take any path arguments.

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--drive-acknowledge-abuse

+

If downloading a file returns the error This file has been identified as malware or spam and cannot be downloaded with the error code cannotDownloadAbusiveFile then supply this flag to rclone to indicate you acknowledge the risks of downloading the file and rclone will download it anyway.

+

--drive-auth-owner-only

+

Only consider files owned by the authenticated user.

+

--drive-chunk-size=SIZE

+

Upload chunk size. Must a power of 2 >= 256k. Default value is 8 MB.

+

Making this larger will improve performance, but note that each chunk is buffered in memory one per transfer.

+

Reducing this will reduce memory usage but decrease performance.

+

--drive-formats

+

Google documents can only be exported from Google drive. When rclone downloads a Google doc it chooses a format to download depending upon this setting.

+

By default the formats are docx,xlsx,pptx,svg which are a sensible default for an editable document.

+

When choosing a format, rclone runs down the list provided in order and chooses the first file format the doc can be exported as from the list. If the file can't be exported to a format on the formats list, then rclone will choose a format from the default list.

+

If you prefer an archive copy then you might use --drive-formats pdf, or if you prefer openoffice/libreoffice formats you might use --drive-formats ods,odt,odp.

+

Note that rclone adds the extension to the google doc, so if it is calles My Spreadsheet on google docs, it will be exported as My Spreadsheet.xlsx or My Spreadsheet.pdf etc.

+

Here are the possible extensions with their corresponding mime types.

+ +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ExtensionMime TypeDescription
csvtext/csvStandard CSV format for Spreadsheets
docapplication/mswordMicosoft Office Document
docxapplication/vnd.openxmlformats-officedocument.wordprocessingml.documentMicrosoft Office Document
epubapplication/epub+zipE-book format
htmltext/htmlAn HTML Document
jpgimage/jpegA JPEG Image File
odpapplication/vnd.oasis.opendocument.presentationOpenoffice Presentation
odsapplication/vnd.oasis.opendocument.spreadsheetOpenoffice Spreadsheet
odsapplication/x-vnd.oasis.opendocument.spreadsheetOpenoffice Spreadsheet
odtapplication/vnd.oasis.opendocument.textOpenoffice Document
pdfapplication/pdfAdobe PDF Format
pngimage/pngPNG Image Format
pptxapplication/vnd.openxmlformats-officedocument.presentationml.presentationMicrosoft Office Powerpoint
rtfapplication/rtfRich Text Format
svgimage/svg+xmlScalable Vector Graphics Format
tsvtext/tab-separated-valuesStandard TSV format for spreadsheets
txttext/plainPlain Text
xlsapplication/vnd.ms-excelMicrosoft Office Spreadsheet
xlsxapplication/vnd.openxmlformats-officedocument.spreadsheetml.sheetMicrosoft Office Spreadsheet
zipapplication/zipA ZIP file of HTML, Images CSS
+

--drive-alternate-export

+

If this option is set this instructs rclone to use an alternate set of export URLs for drive documents. Users have reported that the official export URLs can't export large documents, whereas these unofficial ones can.

+

See rclone issue #2243 for background, this google drive issue and this helpful post.

+

--drive-impersonate user

+

When using a service account, this instructs rclone to impersonate the user passed in.

+

--drive-keep-revision-forever

+

Keeps new head revision of the file forever.

+

--drive-list-chunk int

+

Size of listing chunk 100-1000. 0 to disable. (default 1000)

+

--drive-shared-with-me

+

Instructs rclone to operate on your "Shared with me" folder (where Google Drive lets you access the files and folders others have shared with you).

+

This works both with the "list" (lsd, lsl, etc) and the "copy" commands (copy, sync, etc), and with all other commands too.

+

--drive-skip-gdocs

+

Skip google documents in all listings. If given, gdocs practically become invisible to rclone.

+

--drive-trashed-only

+

Only show files that are in the trash. This will show trashed files in their original directory structure.

+

--drive-upload-cutoff=SIZE

+

File size cutoff for switching to chunked upload. Default is 8 MB.

+

--drive-use-trash

+

Controls whether files are sent to the trash or deleted permanently. Defaults to true, namely sending files to the trash. Use --drive-use-trash=false to delete files permanently instead.

+

--drive-use-created-date

+

Use the file creation date in place of the modification date. Defaults to false.

+

Useful when downloading data and you want the creation date used in place of the last modified date.

+

WARNING: This flag may have some unexpected consequences.

+

When uploading to your drive all files will be overwritten unless they haven't been modified since their creation. And the inverse will occur while downloading. This side effect can be avoided by using the --checksum flag.

+

This feature was implemented to retain photos capture date as recorded by google photos. You will first need to check the "Create a Google Photos folder" option in your google drive settings. You can then copy or move the photos locally and use the date the image was taken (created) set as the modification date.

+

Limitations

+

Drive has quite a lot of rate limiting. This causes rclone to be limited to transferring about 2 files per second only. Individual files may be transferred much faster at 100s of MBytes/s but lots of small files can take a long time.

+

Server side copies are also subject to a separate rate limit. If you see User rate limit exceeded errors, wait at least 24 hours and retry. You can disable server side copies with --disable copy to download and upload the files if you prefer.

+

Limitations of Google Docs

+

Google docs will appear as size -1 in rclone ls and as size 0 in anything which uses the VFS layer, eg rclone mount, rclone serve.

+

This is because rclone can't find out the size of the Google docs without downloading them.

+

Google docs will transfer correctly with rclone sync, rclone copy etc as rclone knows to ignore the size when doing the transfer.

+

However an unfortunate consequence of this is that you can't download Google docs using rclone mount - you will get a 0 sized file. If you try again the doc may gain its correct size and be downloadable.

+

Duplicated files

+

Sometimes, for no reason I've been able to track down, drive will duplicate a file that rclone uploads. Drive unlike all the other remotes can have duplicated files.

+

Duplicated files cause problems with the syncing and you will see messages in the log about duplicates.

+

Use rclone dedupe to fix duplicated files.

+

Note that this isn't just a problem with rclone, even Google Photos on Android duplicates files on drive sometimes.

+

Rclone appears to be re-copying files it shouldn't

+

The most likely cause of this is the duplicated file issue above - run rclone dedupe and check your logs for duplicate object or directory messages.

+

Making your own client_id

+

When you use rclone with Google drive in its default configuration you are using rclone's client_id. This is shared between all the rclone users. There is a global rate limit on the number of queries per second that each client_id can do set by Google. rclone already has a high quota and I will continue to make sure it is high enough by contacting Google.

+

However you might find you get better performance making your own client_id if you are a heavy user. Or you may not depending on exactly how Google have been raising rclone's rate limit.

+

Here is how to create your own Google Drive client ID for rclone:

+
    +

  1. Log into the Google API Console with your Google account. It doesn't matter what Google account you use. (It need not be the same account as the Google Drive you want to access)

    2. +

  2. Select a project or create a new project.

    3. +

  3. Under "ENABLE APIS AND SERVICES" search for "Drive", and enable the then "Google Drive API".

    4. +

  4. Click "Credentials" in the left-side panel (not "Create credentials", which opens the wizard), then "Create credentials", then "OAuth client ID". It will prompt you to set the OAuth consent screen product name, if you haven't set one already.

    5. +

  5. Choose an application type of "other", and click "Create". (the default name is fine)

    6. +

  6. It will show you a client ID and client secret. Use these values in rclone config to add a new remote or edit an existing remote.

    7. +
+

(Thanks to @balazer on github for these instructions.)

+

HTTP

+

The HTTP remote is a read only remote for reading files of a webserver. The webserver should provide file listings which rclone will read and turn into a remote. This has been tested with common webservers such as Apache/Nginx/Caddy and will likely work with file listings from most web servers. (If it doesn't then please file an issue, or send a pull request!)

+

Paths are specified as remote: or remote:path/to/dir.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / FTP Connection
+   \ "ftp"
+ 7 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 8 / Google Drive
+   \ "drive"
+ 9 / Hubic
+   \ "hubic"
+10 / Local Disk
+   \ "local"
+11 / Microsoft OneDrive
+   \ "onedrive"
+12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+13 / SSH/SFTP Connection
+   \ "sftp"
+14 / Yandex Disk
+   \ "yandex"
+15 / http Connection
+   \ "http"
+Storage> http
+URL of http host to connect to
+Choose a number from below, or type in your own value
+ 1 / Connect to example.com
+   \ "https://example.com"
+url> https://beta.rclone.org
+Remote config
+--------------------
+[remote]
+url = https://beta.rclone.org
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+Current remotes:
+
+Name                 Type
+====                 ====
+remote               http
+
+e) Edit existing remote
+n) New remote
+d) Delete remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+e/n/d/r/c/s/q> q
+

This remote is called remote and can now be used like this

+

See all the top level directories

+
rclone lsd remote:
+

List the contents of a directory

+
rclone ls remote:directory
+

Sync the remote directory to /home/local/directory, deleting any excess files.

+
rclone sync remote:directory /home/local/directory
+

Read only

+

This remote is read only - you can't upload files to an HTTP server.

+

Modified time

+

Most HTTP servers store time accurate to 1 second.

+

Checksum

+

No checksums are stored.

+

Usage without a config file

+

Since the http remote only has one config parameter it is easy to use without a config file:

+
rclone lsd --http-url https://beta.rclone.org :http:
+

Hubic

+

Paths are specified as remote:path

+

Paths are specified as remote:container (or remote: for the lsd command.) You may put subdirectories in too, eg remote:container/path/to/dir.

+

The initial setup for Hubic involves getting a token from Hubic which you need to do in your browser. rclone config walks you through it.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
n) New remote
+s) Set configuration password
+n/s> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 7 / Google Drive
+   \ "drive"
+ 8 / Hubic
+   \ "hubic"
+ 9 / Local Disk
+   \ "local"
+10 / Microsoft OneDrive
+   \ "onedrive"
+11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+12 / SSH/SFTP Connection
+   \ "sftp"
+13 / Yandex Disk
+   \ "yandex"
+Storage> 8
+Hubic Client Id - leave blank normally.
+client_id>
+Hubic Client Secret - leave blank normally.
+client_secret>
+Remote config
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine
+y) Yes
+n) No
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+client_id =
+client_secret =
+token = {"access_token":"XXXXXX"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

See the remote setup docs for how to set it up on a machine with no Internet browser available.

+

Note that rclone runs a webserver on your local machine to collect the token as returned from Hubic. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall.

+

Once configured you can then use rclone like this,

+

List containers in the top level of your Hubic

+
rclone lsd remote:
+

List all the files in your Hubic

+
rclone ls remote:
+

To copy a local directory to an Hubic directory called backup

+
rclone copy /home/source remote:backup
+

If you want the directory to be visible in the official Hubic browser, you need to copy your files to the default directory

+
rclone copy /home/source remote:default/backup
+

--fast-list

+

This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.

+

Modified time

+

The modified time is stored as metadata on the object as X-Object-Meta-Mtime as floating point since the epoch accurate to 1 ns.

+

This is a defacto standard (used in the official python-swiftclient amongst others) for storing the modification time for an object.

+

Note that Hubic wraps the Swift backend, so most of the properties of are the same.

+

Limitations

+

This uses the normal OpenStack Swift mechanism to refresh the Swift API credentials and ignores the expires field returned by the Hubic API.

+

The Swift API doesn't return a correct MD5SUM for segmented files (Dynamic or Static Large Objects) so rclone won't check or use the MD5SUM for these.

+

Jottacloud

+

Paths are specified as remote:path

+

Paths may be as deep as required, eg remote:directory/subdirectory.

+

To configure Jottacloud you will need to enter your username and password and select a mountpoint.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+[snip]
+13 / JottaCloud
+   \ "jottacloud"
+[snip]
+Storage> jottacloud
+User Name
+Enter a string value. Press Enter for the default ("").
+user> user
+Password.
+y) Yes type in my own password
+g) Generate random password
+n) No leave this optional password blank
+y/g/n> y
+Enter the password:
+password:
+Confirm the password:
+password:
+The mountpoint to use.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 1 / Will be synced by the official client.
+   \ "Sync"
+ 2 / Archive
+   \ "Archive"
+mountpoint> Archive
+Remote config
+--------------------
+[remote]
+type = jottacloud
+user = user
+pass = *** ENCRYPTED ***
+mountpoint = Archive
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

Once configured you can then use rclone like this,

+

List directories in top level of your Jottacloud

+
rclone lsd remote:
+

List all the files in your Jottacloud

+
rclone ls remote:
+

To copy a local directory to an Jottacloud directory called backup

+
rclone copy /home/source remote:backup
+

Modified time and hashes

+

Jottacloud allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not.

+

Jottacloud supports MD5 type hashes, so you can use the --checksum flag.

+

Note that Jottacloud requires the MD5 hash before upload so if the source does not have an MD5 checksum then the file will be cached temporarily on disk (wherever the TMPDIR environment variable points to) before it is uploaded. Small files will be cached in memory - see the --jottacloud-md5-memory-limit flag.

+

Deleting files

+

Any files you delete with rclone will end up in the trash. Due to a lack of API documentation emptying the trash is currently only possible via the Jottacloud website.

+

Versions

+

Jottacloud supports file versioning. When rclone uploads a new version of a file it creates a new version of it. Currently rclone only supports retrieving the current version but older versions can be accessed via the Jottacloud Website.

+

Limitations

+

Note that Jottacloud is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

+

There are quite a few characters that can't be in Jottacloud file names. Rclone will map these names to and from an identical looking unicode equivalent. For example if a file has a ? in it will be mapped to ？ instead.

+

Jottacloud only supports filenames up to 255 characters in length.

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--jottacloud-md5-memory-limit SizeSuffix

+

Files bigger than this will be cached on disk to calculate the MD5 if required. (default 10M)

+

Troubleshooting

+

Jottacloud exhibits some inconsistent behaviours regarding deleted files and folders which may cause Copy, Move and DirMove operations to previously deleted paths to fail. Emptying the trash should help in such cases.

+

Mega

+

Mega is a cloud storage and file hosting service known for its security feature where all files are encrypted locally before they are uploaded. This prevents anyone (including employees of Mega) from accessing the files without knowledge of the key used for encryption.

+

This is an rclone backend for Mega which supports the file transfer features of Mega using the same client side encryption.

+

Paths are specified as remote:path

+

Paths may be as deep as required, eg remote:directory/subdirectory.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Alias for a existing remote
+   \ "alias"
+[snip]
+14 / Mega
+   \ "mega"
+[snip]
+23 / http Connection
+   \ "http"
+Storage> mega
+User name
+user> you@example.com
+Password.
+y) Yes type in my own password
+g) Generate random password
+n) No leave this optional password blank
+y/g/n> y
+Enter the password:
+password:
+Confirm the password:
+password:
+Remote config
+--------------------
+[remote]
+type = mega
+user = you@example.com
+pass = *** ENCRYPTED ***
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

Once configured you can then use rclone like this,

+

List directories in top level of your Mega

+
rclone lsd remote:
+

List all the files in your Mega

+
rclone ls remote:
+

To copy a local directory to an Mega directory called backup

+
rclone copy /home/source remote:backup
+

Modified time and hashes

+

Mega does not support modification times or hashes yet.

+

Duplicated files

+

Mega can have two files with exactly the same name and path (unlike a normal file system).

+

Duplicated files cause problems with the syncing and you will see messages in the log about duplicates.

+

Use rclone dedupe to fix duplicated files.

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--mega-debug

+

If this flag is set (along with -vv) it will print further debugging information from the mega backend.

+

--mega-hard-delete

+

Normally the mega backend will put all deletions into the trash rather than permanently deleting them. If you specify this flag (or set it in the advanced config) then rclone will permanently delete objects instead.

+

Limitations

+

This backend uses the go-mega go library which is an opensource go library implementing the Mega API. There doesn't appear to be any documentation for the mega protocol beyond the mega C++ SDK source code so there are likely quite a few errors still remaining in this library.

+

Mega allows duplicate files which may confuse rclone.

+

Microsoft Azure Blob Storage

+

Paths are specified as remote:container (or remote: for the lsd command.) You may put subdirectories in too, eg remote:container/path/to/dir.

+

Here is an example of making a Microsoft Azure Blob Storage configuration. For a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Box
+   \ "box"
+ 5 / Dropbox
+   \ "dropbox"
+ 6 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 7 / FTP Connection
+   \ "ftp"
+ 8 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 9 / Google Drive
+   \ "drive"
+10 / Hubic
+   \ "hubic"
+11 / Local Disk
+   \ "local"
+12 / Microsoft Azure Blob Storage
+   \ "azureblob"
+13 / Microsoft OneDrive
+   \ "onedrive"
+14 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+15 / SSH/SFTP Connection
+   \ "sftp"
+16 / Yandex Disk
+   \ "yandex"
+17 / http Connection
+   \ "http"
+Storage> azureblob
+Storage Account Name
+account> account_name
+Storage Account Key
+key> base64encodedkey==
+Endpoint for the service - leave blank normally.
+endpoint> 
+Remote config
+--------------------
+[remote]
+account = account_name
+key = base64encodedkey==
+endpoint = 
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

See all containers

+
rclone lsd remote:
+

Make a new container

+
rclone mkdir remote:container
+

List the contents of a container

+
rclone ls remote:container
+

Sync /home/local/directory to the remote container, deleting any excess files in the container.

+
rclone sync /home/local/directory remote:container
+

--fast-list

+

This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.

+

Modified time

+

The modified time is stored as metadata on the object with the mtime key. It is stored using RFC3339 Format time with nanosecond precision. The metadata is supplied during directory listings so there is no overhead to using it.

+

Hashes

+

MD5 hashes are stored with blobs. However blobs that were uploaded in chunks only have an MD5 if the source remote was capable of MD5 hashes, eg the local disk.

+

Authenticating with Azure Blob Storage

+

Rclone has 3 ways of authenticating with Azure Blob Storage:

+

Account and Key

+

This is the most straight forward and least flexible way. Just fill in the account and key lines and leave the rest blank.

+

SAS URL

+

This can be an account level SAS URL or container level SAS URL

+

To use it leave account, key blank and fill in sas_url.

+

Account level SAS URL or container level SAS URL can be obtained from Azure portal or Azure Storage Explorer. To get a container level SAS URL right click on a container in the Azure Blob explorer in the Azure portal.

+

If You use container level SAS URL, rclone operations are permitted only on particular container, eg

+
rclone ls azureblob:container or rclone ls azureblob:
+

Since container name already exists in SAS URL, you can leave it empty as well.

+

However these will not work

+
rclone lsd azureblob:
+rclone ls azureblob:othercontainer
+

This would be useful for temporarily allowing third parties access to a single container or putting credentials into an untrusted environment.

+

Multipart uploads

+

Rclone supports multipart uploads with Azure Blob storage. Files bigger than 256MB will be uploaded using chunked upload by default.

+

The files will be uploaded in parallel in 4MB chunks (by default). Note that these chunks are buffered in memory and there may be up to --transfers of them being uploaded at once.

+

Files can't be split into more than 50,000 chunks so by default, so the largest file that can be uploaded with 4MB chunk size is 195GB. Above this rclone will double the chunk size until it creates less than 50,000 chunks. By default this will mean a maximum file size of 3.2TB can be uploaded. This can be raised to 5TB using --azureblob-chunk-size 100M.

+

Note that rclone doesn't commit the block list until the end of the upload which means that there is a limit of 9.5TB of multipart uploads in progress as Azure won't allow more than that amount of uncommitted blocks.

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--azureblob-upload-cutoff=SIZE

+

Cutoff for switching to chunked upload - must be <= 256MB. The default is 256MB.

+

--azureblob-chunk-size=SIZE

+

Upload chunk size. Default 4MB. Note that this is stored in memory and there may be up to --transfers chunks stored at once in memory. This can be at most 100MB.

+

--azureblob-access-tier=Hot/Cool/Archive

+

Azure storage supports blob tiering, you can configure tier in advanced settings or supply flag while performing data transfer operations. If there is no access tier specified, rclone doesn't apply any tier. rclone performs Set Tier operation on blobs while uploading, if objects are not modified, specifying access tier to new one will have no effect. If blobs are in archive tier at remote, trying to perform data transfer operations from remote will not be allowed. User should first restore by tiering blob to Hot or Cool.

+

Limitations

+

MD5 sums are only uploaded with chunked files if the source has an MD5 sum. This will always be the case for a local to azure copy.

+

Microsoft OneDrive

+

Paths are specified as remote:path

+

Paths may be as deep as required, eg remote:directory/subdirectory.

+

The initial setup for OneDrive involves getting a token from Microsoft which you need to do in your browser. rclone config walks you through it.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+n/s> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 7 / Google Drive
+   \ "drive"
+ 8 / Hubic
+   \ "hubic"
+ 9 / Local Disk
+   \ "local"
+10 / Microsoft OneDrive
+   \ "onedrive"
+11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+12 / SSH/SFTP Connection
+   \ "sftp"
+13 / Yandex Disk
+   \ "yandex"
+Storage> 10
+Microsoft App Client Id - leave blank normally.
+client_id>
+Microsoft App Client Secret - leave blank normally.
+client_secret>
+Remote config
+Choose OneDrive account type?
+ * Say b for a OneDrive business account
+ * Say p for a personal OneDrive account
+b) Business
+p) Personal
+b/p> p
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine
+y) Yes
+n) No
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+client_id =
+client_secret =
+token = {"access_token":"XXXXXX"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

See the remote setup docs for how to set it up on a machine with no Internet browser available.

+

Note that rclone runs a webserver on your local machine to collect the token as returned from Microsoft. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall.

+

Once configured you can then use rclone like this,

+

List directories in top level of your OneDrive

+
rclone lsd remote:
+

List all the files in your OneDrive

+
rclone ls remote:
+

To copy a local directory to an OneDrive directory called backup

+
rclone copy /home/source remote:backup
+

OneDrive for Business

+

There is additional support for OneDrive for Business. Select "b" when ask

+
Choose OneDrive account type?
+ * Say b for a OneDrive business account
+ * Say p for a personal OneDrive account
+b) Business
+p) Personal
+b/p>
+

After that rclone requires an authentication of your account. The application will first authenticate your account, then query the OneDrive resource URL and do a second (silent) authentication for this resource URL.

+

Modified time and hashes

+

OneDrive allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not.

+

OneDrive personal supports SHA1 type hashes. OneDrive for business and Sharepoint Server support QuickXorHash.

+

For all types of OneDrive you can use the --checksum flag.

+

Deleting files

+

Any files you delete with rclone will end up in the trash. Microsoft doesn't provide an API to permanently delete files, nor to empty the trash, so you will have to do that with one of Microsoft's apps or via the OneDrive website.

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--onedrive-chunk-size=SIZE

+

Above this size files will be chunked - must be multiple of 320k. The default is 10MB. Note that the chunks will be buffered into memory.

+

Limitations

+

Note that OneDrive is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

+

There are quite a few characters that can't be in OneDrive file names. These can't occur on Windows platforms, but on non-Windows platforms they are common. Rclone will map these names to and from an identical looking unicode equivalent. For example if a file has a ? in it will be mapped to ？ instead.

+

The largest allowed file size is 10GiB (10,737,418,240 bytes).

+

Versioning issue

+

Every change in OneDrive causes the service to create a new version. This counts against a users quota. For example changing the modification time of a file creates a second version, so the file is using twice the space.

+

The copy is the only rclone command affected by this as we copy the file and then afterwards set the modification time to match the source file.

+

User Weropol has found a method to disable versioning on OneDrive

+
    +
  1. Open the settings menu by clicking on the gear symbol at the top of the OneDrive Business page.
    2. +
  2. Click Site settings.
    3. +
  3. Once on the Site settings page, navigate to Site Administration > Site libraries and lists.
    4. +
  4. Click Customize "Documents".
    5. +
  5. Click General Settings > Versioning Settings.
    6. +
  6. Under Document Version History select the option No versioning. Note: This will disable the creation of new file versions, but will not remove any previous versions. Your documents are safe.
    7. +
  7. Apply the changes by clicking OK.
    8. +
  8. Use rclone to upload or modify files. (I also use the --no-update-modtime flag)
    9. +
  9. Restore the versioning settings after using rclone. (Optional)
    10. +
+

Troubleshooting

+
Error: access_denied
+Code: AADSTS65005
+Description: Using application 'rclone' is currently not supported for your organization [YOUR_ORGANIZATION] because it is in an unmanaged state. An administrator needs to claim ownership of the company by DNS validation of [YOUR_ORGANIZATION] before the application rclone can be provisioned.
+

This means that rclone can't use the OneDrive for Business API with your account. You can't do much about it, maybe write an email to your admins.

+

However, there are other ways to interact with your OneDrive account. Have a look at the webdav backend: https://rclone.org/webdav/#sharepoint

+

OpenDrive

+

Paths are specified as remote:path

+

Paths may be as deep as required, eg remote:directory/subdirectory.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
n) New remote
+d) Delete remote
+q) Quit config
+e/n/d/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 7 / Google Drive
+   \ "drive"
+ 8 / Hubic
+   \ "hubic"
+ 9 / Local Disk
+   \ "local"
+10 / OpenDrive
+   \ "opendrive"
+11 / Microsoft OneDrive
+   \ "onedrive"
+12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+13 / SSH/SFTP Connection
+   \ "sftp"
+14 / Yandex Disk
+   \ "yandex"
+Storage> 10
+Username
+username>
+Password
+y) Yes type in my own password
+g) Generate random password
+y/g> y
+Enter the password:
+password:
+Confirm the password:
+password:
+--------------------
+[remote]
+username =
+password = *** ENCRYPTED ***
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

List directories in top level of your OpenDrive

+
rclone lsd remote:
+

List all the files in your OpenDrive

+
rclone ls remote:
+

To copy a local directory to an OpenDrive directory called backup

+
rclone copy /home/source remote:backup
+

Modified time and MD5SUMs

+

OpenDrive allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not.

+

Deleting files

+

Any files you delete with rclone will end up in the trash. Amazon don't provide an API to permanently delete files, nor to empty the trash, so you will have to do that with one of Amazon's apps or via the OpenDrive website. As of November 17, 2016, files are automatically deleted by Amazon from the trash after 30 days.

+

Limitations

+

Note that OpenDrive is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

+

There are quite a few characters that can't be in OpenDrive file names. These can't occur on Windows platforms, but on non-Windows platforms they are common. Rclone will map these names to and from an identical looking unicode equivalent. For example if a file has a ? in it will be mapped to ？ instead.

+

QingStor

+

Paths are specified as remote:bucket (or remote: for the lsd command.) You may put subdirectories in too, eg remote:bucket/path/to/dir.

+

Here is an example of making an QingStor configuration. First run

+
rclone config
+

This will guide you through an interactive setup process.

+
No remotes found - make a new one
+n) New remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+n/r/c/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / FTP Connection
+   \ "ftp"
+ 7 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 8 / Google Drive
+   \ "drive"
+ 9 / Hubic
+   \ "hubic"
+10 / Local Disk
+   \ "local"
+11 / Microsoft OneDrive
+   \ "onedrive"
+12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+13 / QingStor Object Storage
+   \ "qingstor"
+14 / SSH/SFTP Connection
+   \ "sftp"
+15 / Yandex Disk
+   \ "yandex"
+Storage> 13
+Get QingStor credentials from runtime. Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own value
+ 1 / Enter QingStor credentials in the next step
+   \ "false"
+ 2 / Get QingStor credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+QingStor Access Key ID - leave blank for anonymous access or runtime credentials.
+access_key_id> access_key
+QingStor Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
+secret_access_key> secret_key
+Enter a endpoint URL to connection QingStor API.
+Leave blank will use the default value "https://qingstor.com:443"
+endpoint>
+Zone connect to. Default is "pek3a".
+Choose a number from below, or type in your own value
+   / The Beijing (China) Three Zone
+ 1 | Needs location constraint pek3a.
+   \ "pek3a"
+   / The Shanghai (China) First Zone
+ 2 | Needs location constraint sh1a.
+   \ "sh1a"
+zone> 1
+Number of connnection retry.
+Leave blank will use the default value "3".
+connection_retries>
+Remote config
+--------------------
+[remote]
+env_auth = false
+access_key_id = access_key
+secret_access_key = secret_key
+endpoint =
+zone = pek3a
+connection_retries =
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

This remote is called remote and can now be used like this

+

See all buckets

+
rclone lsd remote:
+

Make a new bucket

+
rclone mkdir remote:bucket
+

List the contents of a bucket

+
rclone ls remote:bucket
+

Sync /home/local/directory to the remote bucket, deleting any excess files in the bucket.

+
rclone sync /home/local/directory remote:bucket
+

--fast-list

+

This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.

+

Multipart uploads

+

rclone supports multipart uploads with QingStor which means that it can upload files bigger than 5GB. Note that files uploaded with multipart upload don't have an MD5SUM.

+

Buckets and Zone

+

With QingStor you can list buckets (rclone lsd) using any zone, but you can only access the content of a bucket from the zone it was created in. If you attempt to access a bucket from the wrong zone, you will get an error, incorrect zone, the bucket is not in 'XXX' zone.

+

Authentication

+

There are two ways to supply rclone with a set of QingStor credentials. In order of precedence:

+ +

Swift

+

Swift refers to Openstack Object Storage. Commercial implementations of that being:

+ +

Paths are specified as remote:container (or remote: for the lsd command.) You may put subdirectories in too, eg remote:container/path/to/dir.

+

Here is an example of making a swift configuration. First run

+
rclone config
+

This will guide you through an interactive setup process.

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Box
+   \ "box"
+ 5 / Cache a remote
+   \ "cache"
+ 6 / Dropbox
+   \ "dropbox"
+ 7 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 8 / FTP Connection
+   \ "ftp"
+ 9 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+10 / Google Drive
+   \ "drive"
+11 / Hubic
+   \ "hubic"
+12 / Local Disk
+   \ "local"
+13 / Microsoft Azure Blob Storage
+   \ "azureblob"
+14 / Microsoft OneDrive
+   \ "onedrive"
+15 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+16 / Pcloud
+   \ "pcloud"
+17 / QingCloud Object Storage
+   \ "qingstor"
+18 / SSH/SFTP Connection
+   \ "sftp"
+19 / Webdav
+   \ "webdav"
+20 / Yandex Disk
+   \ "yandex"
+21 / http Connection
+   \ "http"
+Storage> swift
+Get swift credentials from environment variables in standard OpenStack form.
+Choose a number from below, or type in your own value
+ 1 / Enter swift credentials in the next step
+   \ "false"
+ 2 / Get swift credentials from environment vars. Leave other fields blank if using this.
+   \ "true"
+env_auth> true
+User name to log in (OS_USERNAME).
+user> 
+API key or password (OS_PASSWORD).
+key> 
+Authentication URL for server (OS_AUTH_URL).
+Choose a number from below, or type in your own value
+ 1 / Rackspace US
+   \ "https://auth.api.rackspacecloud.com/v1.0"
+ 2 / Rackspace UK
+   \ "https://lon.auth.api.rackspacecloud.com/v1.0"
+ 3 / Rackspace v2
+   \ "https://identity.api.rackspacecloud.com/v2.0"
+ 4 / Memset Memstore UK
+   \ "https://auth.storage.memset.com/v1.0"
+ 5 / Memset Memstore UK v2
+   \ "https://auth.storage.memset.com/v2.0"
+ 6 / OVH
+   \ "https://auth.cloud.ovh.net/v2.0"
+auth> 
+User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID).
+user_id> 
+User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME)
+domain> 
+Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME)
+tenant> 
+Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID)
+tenant_id> 
+Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME)
+tenant_domain> 
+Region name - optional (OS_REGION_NAME)
+region> 
+Storage URL - optional (OS_STORAGE_URL)
+storage_url> 
+Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)
+auth_token> 
+AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)
+auth_version> 
+Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE)
+Choose a number from below, or type in your own value
+ 1 / Public (default, choose this if not sure)
+   \ "public"
+ 2 / Internal (use internal service net)
+   \ "internal"
+ 3 / Admin
+   \ "admin"
+endpoint_type> 
+Remote config
+--------------------
+[test]
+env_auth = true
+user = 
+key = 
+auth = 
+user_id = 
+domain = 
+tenant = 
+tenant_id = 
+tenant_domain = 
+region = 
+storage_url = 
+auth_token = 
+auth_version = 
+endpoint_type = 
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

This remote is called remote and can now be used like this

+

See all containers

+
rclone lsd remote:
+

Make a new container

+
rclone mkdir remote:container
+

List the contents of a container

+
rclone ls remote:container
+

Sync /home/local/directory to the remote container, deleting any excess files in the container.

+
rclone sync /home/local/directory remote:container
+

Configuration from an Openstack credentials file

+

An Opentstack credentials file typically looks something something like this (without the comments)

+
export OS_AUTH_URL=https://a.provider.net/v2.0
+export OS_TENANT_ID=ffffffffffffffffffffffffffffffff
+export OS_TENANT_NAME="1234567890123456"
+export OS_USERNAME="123abc567xy"
+echo "Please enter your OpenStack Password: "
+read -sr OS_PASSWORD_INPUT
+export OS_PASSWORD=$OS_PASSWORD_INPUT
+export OS_REGION_NAME="SBG1"
+if [ -z "$OS_REGION_NAME" ]; then unset OS_REGION_NAME; fi
+

The config file needs to look something like this where $OS_USERNAME represents the value of the OS_USERNAME variable - 123abc567xy in the example above.

+
[remote]
+type = swift
+user = $OS_USERNAME
+key = $OS_PASSWORD
+auth = $OS_AUTH_URL
+tenant = $OS_TENANT_NAME
+

Note that you may (or may not) need to set region too - try without first.

+

Configuration from the environment

+

If you prefer you can configure rclone to use swift using a standard set of OpenStack environment variables.

+

When you run through the config, make sure you choose true for env_auth and leave everything else blank.

+

rclone will then set any empty config parameters from the environment using standard OpenStack environment variables. There is a list of the variables in the docs for the swift library.

+

Using an alternate authentication method

+

If your OpenStack installation uses a non-standard authentication method that might not be yet supported by rclone or the underlying swift library, you can authenticate externally (e.g. calling manually the openstack commands to get a token). Then, you just need to pass the two configuration variables auth_token and storage_url. If they are both provided, the other variables are ignored. rclone will not try to authenticate but instead assume it is already authenticated and use these two variables to access the OpenStack installation.

+

Using rclone without a config file

+

You can use rclone with swift without a config file, if desired, like this:

+
source openstack-credentials-file
+export RCLONE_CONFIG_MYREMOTE_TYPE=swift
+export RCLONE_CONFIG_MYREMOTE_ENV_AUTH=true
+rclone lsd myremote:
+

--fast-list

+

This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.

+

--update and --use-server-modtime

+

As noted below, the modified time is stored on metadata on the object. It is used by default for all operations that require checking the time a file was last updated. It allows rclone to treat the remote more like a true filesystem, but it is inefficient because it requires an extra API call to retrieve the metadata.

+

For many operations, the time the object was last uploaded to the remote is sufficient to determine if it is "dirty". By using --update along with --use-server-modtime, you can avoid the extra API call and simply upload files whose local modtime is newer than the time it was last uploaded.

+

Specific options

+

Here are the command line options specific to this cloud storage system.

+

--swift-storage-policy=STRING

+

Apply the specified storage policy when creating a new container. The policy cannot be changed afterwards. The allowed configuration values and their meaning depend on your Swift storage provider.

+

--swift-chunk-size=SIZE

+

Above this size files will be chunked into a _segments container. The default for this is 5GB which is its maximum value.

+

Modified time

+

The modified time is stored as metadata on the object as X-Object-Meta-Mtime as floating point since the epoch accurate to 1 ns.

+

This is a defacto standard (used in the official python-swiftclient amongst others) for storing the modification time for an object.

+

Limitations

+

The Swift API doesn't return a correct MD5SUM for segmented files (Dynamic or Static Large Objects) so rclone won't check or use the MD5SUM for these.

+

Troubleshooting

+

Rclone gives Failed to create file system for "remote:": Bad Request

+

Due to an oddity of the underlying swift library, it gives a "Bad Request" error rather than a more sensible error when the authentication fails for Swift.

+

So this most likely means your username / password is wrong. You can investigate further with the --dump-bodies flag.

+

This may also be caused by specifying the region when you shouldn't have (eg OVH).

+

Rclone gives Failed to create file system: Response didn't have storage storage url and auth token

+

This is most likely caused by forgetting to specify your tenant when setting up a swift remote.

+

pCloud

+

Paths are specified as remote:path

+

Paths may be as deep as required, eg remote:directory/subdirectory.

+

The initial setup for pCloud involves getting a token from pCloud which you need to do in your browser. rclone config walks you through it.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Box
+   \ "box"
+ 5 / Dropbox
+   \ "dropbox"
+ 6 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 7 / FTP Connection
+   \ "ftp"
+ 8 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 9 / Google Drive
+   \ "drive"
+10 / Hubic
+   \ "hubic"
+11 / Local Disk
+   \ "local"
+12 / Microsoft Azure Blob Storage
+   \ "azureblob"
+13 / Microsoft OneDrive
+   \ "onedrive"
+14 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+15 / Pcloud
+   \ "pcloud"
+16 / QingCloud Object Storage
+   \ "qingstor"
+17 / SSH/SFTP Connection
+   \ "sftp"
+18 / Yandex Disk
+   \ "yandex"
+19 / http Connection
+   \ "http"
+Storage> pcloud
+Pcloud App Client Id - leave blank normally.
+client_id> 
+Pcloud App Client Secret - leave blank normally.
+client_secret> 
+Remote config
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine
+y) Yes
+n) No
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+client_id = 
+client_secret = 
+token = {"access_token":"XXX","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

See the remote setup docs for how to set it up on a machine with no Internet browser available.

+

Note that rclone runs a webserver on your local machine to collect the token as returned from pCloud. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall.

+

Once configured you can then use rclone like this,

+

List directories in top level of your pCloud

+
rclone lsd remote:
+

List all the files in your pCloud

+
rclone ls remote:
+

To copy a local directory to an pCloud directory called backup

+
rclone copy /home/source remote:backup
+

Modified time and hashes

+

pCloud allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not. In order to set a Modification time pCloud requires the object be re-uploaded.

+

pCloud supports MD5 and SHA1 type hashes, so you can use the --checksum flag.

+

Deleting files

+

Deleted files will be moved to the trash. Your subscription level will determine how long items stay in the trash. rclone cleanup can be used to empty the trash.

+

SFTP

+

SFTP is the Secure (or SSH) File Transfer Protocol.

+

SFTP runs over SSH v2 and is installed as standard with most modern SSH installations.

+

Paths are specified as remote:path. If the path does not begin with a / it is relative to the home directory of the user. An empty path remote: refers to the user's home directory.

+

Note that some SFTP servers will need the leading / - Synology is a good example of this.

+

Here is an example of making an SFTP configuration. First run

+
rclone config
+

This will guide you through an interactive setup process.

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / FTP Connection
+   \ "ftp"
+ 7 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 8 / Google Drive
+   \ "drive"
+ 9 / Hubic
+   \ "hubic"
+10 / Local Disk
+   \ "local"
+11 / Microsoft OneDrive
+   \ "onedrive"
+12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+13 / SSH/SFTP Connection
+   \ "sftp"
+14 / Yandex Disk
+   \ "yandex"
+15 / http Connection
+   \ "http"
+Storage> sftp
+SSH host to connect to
+Choose a number from below, or type in your own value
+ 1 / Connect to example.com
+   \ "example.com"
+host> example.com
+SSH username, leave blank for current username, ncw
+user> sftpuser
+SSH port, leave blank to use default (22)
+port> 
+SSH password, leave blank to use ssh-agent.
+y) Yes type in my own password
+g) Generate random password
+n) No leave this optional password blank
+y/g/n> n
+Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.
+key_file> 
+Remote config
+--------------------
+[remote]
+host = example.com
+user = sftpuser
+port = 
+pass = 
+key_file = 
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

This remote is called remote and can now be used like this:

+

See all directories in the home directory

+
rclone lsd remote:
+

Make a new directory

+
rclone mkdir remote:path/to/directory
+

List the contents of a directory

+
rclone ls remote:path/to/directory
+

Sync /home/local/directory to the remote directory, deleting any excess files in the directory.

+
rclone sync /home/local/directory remote:directory
+

SSH Authentication

+

The SFTP remote supports three authentication methods:

+ +

Key files should be unencrypted PEM-encoded private key files. For instance /home/$USER/.ssh/id_rsa.

+

If you don't specify pass or key_file then rclone will attempt to contact an ssh-agent.

+

If you set the --sftp-ask-password option, rclone will prompt for a password when needed and no password has been configured.

+

ssh-agent on macOS

+

Note that there seem to be various problems with using an ssh-agent on macOS due to recent changes in the OS. The most effective work-around seems to be to start an ssh-agent in each session, eg

+
eval `ssh-agent -s` && ssh-add -A
+

And then at the end of the session

+
eval `ssh-agent -k`
+

These commands can be used in scripts of course.

+

Specific options

+

Here are the command line options specific to this remote.

+

--sftp-ask-password

+

Ask for the SFTP password if needed when no password has been configured.

+

--ssh-path-override

+

Override path used by SSH connection. Allows checksum calculation when SFTP and SSH paths are different. This issue affects among others Synology NAS boxes.

+

Shared folders can be found in directories representing volumes

+
rclone sync /home/local/directory remote:/directory --ssh-path-override /volume2/directory
+

Home directory can be found in a shared folder called homes

+
rclone sync /home/local/directory remote:/home/directory --ssh-path-override /volume1/homes/USER/directory
+

Modified time

+

Modified times are stored on the server to 1 second precision.

+

Modified times are used in syncing and are fully supported.

+

Some SFTP servers disable setting/modifying the file modification time after upload (for example, certain configurations of ProFTPd with mod_sftp). If you are using one of these servers, you can set the option set_modtime = false in your RClone backend configuration to disable this behaviour.

+

Limitations

+

SFTP supports checksums if the same login has shell access and md5sum or sha1sum as well as echo are in the remote's PATH. This remote checksumming (file hashing) is recommended and enabled by default. Disabling the checksumming may be required if you are connecting to SFTP servers which are not under your control, and to which the execution of remote commands is prohibited. Set the configuration option disable_hashcheck to true to disable checksumming.

+

Note that some SFTP servers (eg Synology) the paths are different for SSH and SFTP so the hashes can't be calculated properly. For them using disable_hashcheck is a good idea.

+

The only ssh agent supported under Windows is Putty's pageant.

+

The Go SSH library disables the use of the aes128-cbc cipher by default, due to security concerns. This can be re-enabled on a per-connection basis by setting the use_insecure_cipher setting in the configuration file to true. Further details on the insecurity of this cipher can be found [in this paper] (http://www.isg.rhul.ac.uk/~kp/SandPfinal.pdf).

+

SFTP isn't supported under plan9 until this issue is fixed.

+

Note that since SFTP isn't HTTP based the following flags don't work with it: --dump-headers, --dump-bodies, --dump-auth

+

Note that --timeout isn't supported (but --contimeout is).

+

WebDAV

+

Paths are specified as remote:path

+

Paths may be as deep as required, eg remote:directory/subdirectory.

+

To configure the WebDAV remote you will need to have a URL for it, and a username and password. If you know what kind of system you are connecting to then rclone can enable extra features.

+

Here is an example of how to make a remote called remote. First run:

+
 rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+[snip]
+22 / Webdav
+   \ "webdav"
+[snip]
+Storage> webdav
+URL of http host to connect to
+Choose a number from below, or type in your own value
+ 1 / Connect to example.com
+   \ "https://example.com"
+url> https://example.com/remote.php/webdav/
+Name of the Webdav site/service/software you are using
+Choose a number from below, or type in your own value
+ 1 / Nextcloud
+   \ "nextcloud"
+ 2 / Owncloud
+   \ "owncloud"
+ 3 / Sharepoint
+   \ "sharepoint"
+ 4 / Other site/service or software
+   \ "other"
+vendor> 1
+User name
+user> user
+Password.
+y) Yes type in my own password
+g) Generate random password
+n) No leave this optional password blank
+y/g/n> y
+Enter the password:
+password:
+Confirm the password:
+password:
+Bearer token instead of user/pass (eg a Macaroon)
+bearer_token> 
+Remote config
+--------------------
+[remote]
+type = webdav
+url = https://example.com/remote.php/webdav/
+vendor = nextcloud
+user = user
+pass = *** ENCRYPTED ***
+bearer_token = 
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

Once configured you can then use rclone like this,

+

List directories in top level of your WebDAV

+
rclone lsd remote:
+

List all the files in your WebDAV

+
rclone ls remote:
+

To copy a local directory to an WebDAV directory called backup

+
rclone copy /home/source remote:backup
+

Modified time and hashes

+

Plain WebDAV does not support modified times. However when used with Owncloud or Nextcloud rclone will support modified times.

+

Hashes are not supported.

+

Provider notes

+

See below for notes on specific providers.

+

Owncloud

+

Click on the settings cog in the bottom right of the page and this will show the WebDAV URL that rclone needs in the config step. It will look something like https://example.com/remote.php/webdav/.

+

Owncloud supports modified times using the X-OC-Mtime header.

+

Nextcloud

+

This is configured in an identical way to Owncloud. Note that Nextcloud does not support streaming of files (rcat) whereas Owncloud does. This may be fixed in the future.

+

Put.io

+

put.io can be accessed in a read only way using webdav.

+

Configure the url as https://webdav.put.io and use your normal account username and password for user and pass. Set the vendor to other.

+

Your config file should end up looking like this:

+
[putio]
+type = webdav
+url = https://webdav.put.io
+vendor = other
+user = YourUserName
+pass = encryptedpassword
+

If you are using put.io with rclone mount then use the --read-only flag to signal to the OS that it can't write to the mount.

+

For more help see the put.io webdav docs.

+

Sharepoint

+

Rclone can be used with Sharepoint provided by OneDrive for Business or Office365 Education Accounts. This feature is only needed for a few of these Accounts, mostly Office365 Education ones. These accounts are sometimes not verified by the domain owner github#1975

+

This means that these accounts can't be added using the official API (other Accounts should work with the "onedrive" option). However, it is possible to access them using webdav.

+

To use a sharepoint remote with rclone, add it like this: First, you need to get your remote's URL:

+ +

You'll only need this URL upto the email address. After that, you'll most likely want to add "/Documents". That subdirectory contains the actual data stored on your OneDrive.

+

Add the remote to rclone like this: Configure the url as https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/Documents and use your normal account email and password for user and pass. If you have 2FA enabled, you have to generate an app password. Set the vendor to sharepoint.

+

Your config file should look like this:

+
[sharepoint]
+type = webdav
+url = https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/Documents
+vendor = other
+user = YourEmailAddress
+pass = encryptedpassword
+

dCache

+

dCache is a storage system with WebDAV doors that support, beside basic and x509, authentication with Macaroons (bearer tokens).

+

Configure as normal using the other type. Don't enter a username or password, instead enter your Macaroon as the bearer_token.

+

The config will end up looking something like this.

+
[dcache]
+type = webdav
+url = https://dcache...
+vendor = other
+user =
+pass =
+bearer_token = your-macaroon
+

There is a script that obtains a Macaroon from a dCache WebDAV endpoint, and creates an rclone config file.

+

Yandex Disk

+

Yandex Disk is a cloud storage solution created by Yandex.

+

Yandex paths may be as deep as required, eg remote:directory/subdirectory.

+

Here is an example of making a yandex configuration. First run

+
rclone config
+

This will guide you through an interactive setup process:

+
No remotes found - make a new one
+n) New remote
+s) Set configuration password
+n/s> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 7 / Google Drive
+   \ "drive"
+ 8 / Hubic
+   \ "hubic"
+ 9 / Local Disk
+   \ "local"
+10 / Microsoft OneDrive
+   \ "onedrive"
+11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+12 / SSH/SFTP Connection
+   \ "sftp"
+13 / Yandex Disk
+   \ "yandex"
+Storage> 13
+Yandex Client Id - leave blank normally.
+client_id>
+Yandex Client Secret - leave blank normally.
+client_secret>
+Remote config
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine
+y) Yes
+n) No
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+client_id =
+client_secret =
+token = {"access_token":"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","token_type":"bearer","expiry":"2016-12-29T12:27:11.362788025Z"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+

See the remote setup docs for how to set it up on a machine with no Internet browser available.

+

Note that rclone runs a webserver on your local machine to collect the token as returned from Yandex Disk. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall.

+

Once configured you can then use rclone like this,

+

See top level directories

+
rclone lsd remote:
+

Make a new directory

+
rclone mkdir remote:directory
+

List the contents of a directory

+
rclone ls remote:directory
+

Sync /home/local/directory to the remote path, deleting any excess files in the path.

+
rclone sync /home/local/directory remote:directory
+

--fast-list

+

This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.

+

Modified time

+

Modified times are supported and are stored accurate to 1 ns in custom metadata called rclone_modified in RFC3339 with nanoseconds format.

+

MD5 checksums

+

MD5 checksums are natively supported by Yandex Disk.

+

Emptying Trash

+

If you wish to empty your trash you can use the rclone cleanup remote: command which will permanently delete all your trashed files. This command does not take any path arguments.

+

Local Filesystem

+

Local paths are specified as normal filesystem paths, eg /path/to/wherever, so

+
rclone sync /home/source /tmp/destination
+

Will sync /home/source to /tmp/destination

+

These can be configured into the config file for consistencies sake, but it is probably easier not to.

+

Modified time

+

Rclone reads and writes the modified time using an accuracy determined by the OS. Typically this is 1ns on Linux, 10 ns on Windows and 1 Second on OS X.

+

Filenames

+

Filenames are expected to be encoded in UTF-8 on disk. This is the normal case for Windows and OS X.

+

There is a bit more uncertainty in the Linux world, but new distributions will have UTF-8 encoded files names. If you are using an old Linux filesystem with non UTF-8 file names (eg latin1) then you can use the convmv tool to convert the filesystem to UTF-8. This tool is available in most distributions' package managers.

+

If an invalid (non-UTF8) filename is read, the invalid characters will be replaced with the unicode replacement character, '�'. rclone will emit a debug message in this case (use -v to see), eg

+
Local file system at .: Replacing invalid UTF-8 characters in "gro\xdf"
+

Long paths on Windows

+

Rclone handles long paths automatically, by converting all paths to long UNC paths which allows paths up to 32,767 characters.

+

This is why you will see that your paths, for instance c:\files is converted to the UNC path \\?\c:\files in the output, and \\server\share is converted to \\?\UNC\server\share.

+

However, in rare cases this may cause problems with buggy file system drivers like EncFS. To disable UNC conversion globally, add this to your .rclone.conf file:

+
[local]
+nounc = true
+

If you want to selectively disable UNC, you can add it to a separate entry like this:

+
[nounc]
+type = local
+nounc = true
+

And use rclone like this:

+

rclone copy c:\src nounc:z:\dst

+

This will use UNC paths on c:\src but not on z:\dst. Of course this will cause problems if the absolute path length of a file exceeds 258 characters on z, so only use this option if you have to.

+

Specific options

+

Here are the command line options specific to local storage

+

--copy-links, -L

+

Normally rclone will ignore symlinks or junction points (which behave like symlinks under Windows).

+

If you supply this flag then rclone will follow the symlink and copy the pointed to file or directory.

+

This flag applies to all commands.

+

For example, supposing you have a directory structure like this

+
$ tree /tmp/a
+/tmp/a
+├── b -> ../b
+├── expected -> ../expected
+├── one
+└── two
+    └── three
+

Then you can see the difference with and without the flag like this

+
$ rclone ls /tmp/a
+        6 one
+        6 two/three
+

and

+
$ rclone -L ls /tmp/a
+     4174 expected
+        6 one
+        6 two/three
+        6 b/two
+        6 b/one
+

--local-no-check-updated

+

Don't check to see if the files change during upload.

+

Normally rclone checks the size and modification time of files as they are being uploaded and aborts with a message which starts can't copy - source file is being updated if the file changes during upload.

+

However on some file systems this modification time check may fail (eg Glusterfs #2206) so this check can be disabled with this flag.

+

--local-no-unicode-normalization

+

This flag is deprecated now. Rclone no longer normalizes unicode file names, but it compares them with unicode normalization in the sync routine instead.

+

--one-file-system, -x

+

This tells rclone to stay in the filesystem specified by the root and not to recurse into different file systems.

+

For example if you have a directory hierarchy like this

+
root
+├── disk1     - disk1 mounted on the root
+│   └── file3 - stored on disk1
+├── disk2     - disk2 mounted on the root
+│   └── file4 - stored on disk12
+├── file1     - stored on the root disk
+└── file2     - stored on the root disk
+

Using rclone --one-file-system copy root remote: will only copy file1 and file2. Eg

+
$ rclone -q --one-file-system ls root
+        0 file1
+        0 file2
+
$ rclone -q ls root
+        0 disk1/file3
+        0 disk2/file4
+        0 file1
+        0 file2
+

NB Rclone (like most unix tools such as du, rsync and tar) treats a bind mount to the same device as being on the same filesystem.

+

NB This flag is only available on Unix based systems. On systems where it isn't supported (eg Windows) it will not appear as an valid flag.

+ +

This flag disables warning messages on skipped symlinks or junction points, as you explicitly acknowledge that they should be skipped.

+

Changelog

+

v1.43.1 - 2018-09-07

+

Point release to fix hubic and azureblob backends.

+ +

v1.43 - 2018-09-01

+ +

v1.42 - 2018-06-16

+ +

v1.41 - 2018-04-28

+ +

v1.40 - 2018-03-19

+ +

v1.39 - 2017-12-23

+ +

v1.38 - 2017-09-30

+ +

v1.37 - 2017-07-22

+ +

v1.36 - 2017-03-18

+ +

v1.35 - 2017-01-02

+ +

v1.34 - 2016-11-06

+ +

v1.33 - 2016-08-24

+ +

v1.32 - 2016-07-13

+ +

v1.31 - 2016-07-13

+ +

v1.30 - 2016-06-18

+ +

v1.29 - 2016-04-18

+ +

v1.28 - 2016-03-01

+ +

v1.27 - 2016-01-31

+ +

v1.26 - 2016-01-02

+ +

v1.25 - 2015-11-14

+ +

v1.24 - 2015-11-07

+ +

v1.23 - 2015-10-03

+ +

v1.22 - 2015-09-28

+ +

v1.21 - 2015-09-22

+ +

v1.20 - 2015-09-15

+ +

v1.19 - 2015-08-28

+ +

v1.18 - 2015-08-17

+ +

v1.17 - 2015-06-14

+ +

v1.16 - 2015-06-09

+ +

v1.15 - 2015-06-06

+ +

v1.14 - 2015-05-21

+ +

v1.13 - 2015-05-10

+ +

v1.12 - 2015-03-15

+ +

v1.11 - 2015-03-04

+ +

v1.10 - 2015-02-12

+ +

v1.09 - 2015-02-07

+ +

v1.08 - 2015-02-04

+ +

v1.07 - 2014-12-23

+ +

v1.06 - 2014-12-12

+ +

v1.05 - 2014-08-09

+ +

v1.04 - 2014-07-21

+ +

v1.03 - 2014-07-20

+ +

v1.02 - 2014-07-19

+ +

v1.01 - 2014-07-04

+ +

v1.00 - 2014-07-03

+ +

v0.99 - 2014-06-26

+ +

v0.98 - 2014-05-30

+ +

v0.97 - 2014-05-05

+ +

v0.96 - 2014-04-24

+ +

v0.95 - 2014-03-28

+ +

v0.94 - 2014-03-27

+ +

v0.93 - 2014-03-16

+ +

v0.92 - 2014-03-15

+ +

v0.91 - 2014-03-15

+ +

v0.90 - 2013-06-27

+ +

v0.00 - 2012-11-18

+ +

Bugs and Limitations

+

Empty directories are left behind / not created

+

With remotes that have a concept of directory, eg Local and Drive, empty directories may be left behind, or not created when one was expected.

+

This is because rclone doesn't have a concept of a directory - it only works on objects. Most of the object storage systems can't actually store a directory so there is nowhere for rclone to store anything about directories.

+

You can work round this to some extent with thepurge command which will delete everything under the path, inluding empty directories.

+

This may be fixed at some point in Issue #100

+

Directory timestamps aren't preserved

+

For the same reason as the above, rclone doesn't have a concept of a directory - it only works on objects, therefore it can't preserve the timestamps of directories.

+

Frequently Asked Questions

+

Do all cloud storage systems support all rclone commands

+

Yes they do. All the rclone commands (eg sync, copy etc) will work on all the remote storage systems.

+

Can I copy the config from one machine to another

+

Sure! Rclone stores all of its config in a single file. If you want to find this file, the simplest way is to run rclone -h and look at the help for the --config flag which will tell you where it is.

+

See the remote setup docs for more info.

+

How do I configure rclone on a remote / headless box with no browser?

+

This has now been documented in its own remote setup page.

+

Can rclone sync directly from drive to s3

+

Rclone can sync between two remote cloud storage systems just fine.

+

Note that it effectively downloads the file and uploads it again, so the node running rclone would need to have lots of bandwidth.

+

The syncs would be incremental (on a file by file basis).

+

Eg

+
rclone sync drive:Folder s3:bucket
+

Using rclone from multiple locations at the same time

+

You can use rclone from multiple places at the same time if you choose different subdirectory for the output, eg

+
Server A> rclone sync /tmp/whatever remote:ServerA
+Server B> rclone sync /tmp/whatever remote:ServerB
+

If you sync to the same directory then you should use rclone copy otherwise the two rclones may delete each others files, eg

+
Server A> rclone copy /tmp/whatever remote:Backup
+Server B> rclone copy /tmp/whatever remote:Backup
+

The file names you upload from Server A and Server B should be different in this case, otherwise some file systems (eg Drive) may make duplicates.

+

Why doesn't rclone support partial transfers / binary diffs like rsync?

+

Rclone stores each file you transfer as a native object on the remote cloud storage system. This means that you can see the files you upload as expected using alternative access methods (eg using the Google Drive web interface). There is a 1:1 mapping between files on your hard disk and objects created in the cloud storage system.

+

Cloud storage systems (at least none I've come across yet) don't support partially uploading an object. You can't take an existing object, and change some bytes in the middle of it.

+

It would be possible to make a sync system which stored binary diffs instead of whole objects like rclone does, but that would break the 1:1 mapping of files on your hard disk to objects in the remote cloud storage system.

+

All the cloud storage systems support partial downloads of content, so it would be possible to make partial downloads work. However to make this work efficiently this would require storing a significant amount of metadata, which breaks the desired 1:1 mapping of files to objects.

+

Can rclone do bi-directional sync?

+

No, not at present. rclone only does uni-directional sync from A -> B. It may do in the future though since it has all the primitives - it just requires writing the algorithm to do it.

+

Can I use rclone with an HTTP proxy?

+

Yes. rclone will use the environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY, similar to cURL and other programs.

+

HTTPS_PROXY takes precedence over HTTP_PROXY for https requests.

+

The environment values may be either a complete URL or a "host[:port]", in which case the "http" scheme is assumed.

+

The NO_PROXY allows you to disable the proxy for specific hosts. Hosts must be comma separated, and can contain domains or parts. For instance "foo.com" also matches "bar.foo.com".

+

Rclone gives x509: failed to load system roots and no roots provided error

+

This means that rclone can't file the SSL root certificates. Likely you are running rclone on a NAS with a cut-down Linux OS, or possibly on Solaris.

+

Rclone (via the Go runtime) tries to load the root certificates from these places on Linux.

+
"/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc.
+"/etc/pki/tls/certs/ca-bundle.crt",   // Fedora/RHEL
+"/etc/ssl/ca-bundle.pem",             // OpenSUSE
+"/etc/pki/tls/cacert.pem",            // OpenELEC
+

So doing something like this should fix the problem. It also sets the time which is important for SSL to work properly.

+
mkdir -p /etc/ssl/certs/
+curl -o /etc/ssl/certs/ca-certificates.crt https://raw.githubusercontent.com/bagder/ca-bundle/master/ca-bundle.crt
+ntpclient -s -h pool.ntp.org
+

The two environment variables SSL_CERT_FILE and SSL_CERT_DIR, mentioned in the x509 pacakge, provide an additional way to provide the SSL root certificates.

+

Note that you may need to add the --insecure option to the curl command line if it doesn't work without.

+
curl --insecure -o /etc/ssl/certs/ca-certificates.crt https://raw.githubusercontent.com/bagder/ca-bundle/master/ca-bundle.crt
+

Rclone gives Failed to load config file: function not implemented error

+

Likely this means that you are running rclone on Linux version not supported by the go runtime, ie earlier than version 2.6.23.

+

See the system requirements section in the go install docs for full details.

+

All my uploaded docx/xlsx/pptx files appear as archive/zip

+

This is caused by uploading these files from a Windows computer which hasn't got the Microsoft Office suite installed. The easiest way to fix is to install the Word viewer and the Microsoft Office Compatibility Pack for Word, Excel, and PowerPoint 2007 and later versions' file formats

+

tcp lookup some.domain.com no such host

+

This happens when rclone cannot resolve a domain. Please check that your DNS setup is generally working, e.g.

+
# both should print a long list of possible IP addresses
+dig www.googleapis.com          # resolve using your default DNS
+dig www.googleapis.com @8.8.8.8 # resolve with Google's DNS server
+

If you are using systemd-resolved (default on Arch Linux), ensure it is at version 233 or higher. Previous releases contain a bug which causes not all domains to be resolved properly.

+

Additionally with the GODEBUG=netdns= environment variable the Go resolver decision can be influenced. This also allows to resolve certain issues with DNS resolution. See the name resolution section in the go docs.

+

License

+

This is free software under the terms of MIT the license (check the COPYING file included with the source code).

+
Copyright (C) 2012 by Nick Craig-Wood https://www.craig-wood.com/nick/
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

Authors

+ +

Contributors

+ +

Contact the rclone project

+

Forum

+

Forum for general discussions and questions:

+ +

Gitub project

+

The project website is at:

+ +

There you can file bug reports, ask for help or contribute pull requests.

+

Google+

+

Rclone has a Google+ page which announcements are posted to

+ +

Twitter

+

You can also follow me on twitter for rclone announcements

+ +

Email

+

Or if all else fails or you want to ask something private or confidential email Nick Craig-Wood

+ + diff --git a/.rclone_repo/MANUAL.md b/.rclone_repo/MANUAL.md new file mode 100755 index 0000000..9c44e40 --- /dev/null +++ b/.rclone_repo/MANUAL.md @@ -0,0 +1,13459 @@ +% rclone(1) User Manual +% Nick Craig-Wood +% Sep 07, 2018 + +Rclone +====== + +[![Logo](https://rclone.org/img/rclone-120x120.png)](https://rclone.org/) + +Rclone is a command line program to sync files and directories to and from: + +* Amazon Drive ([See note](/amazonclouddrive/#status)) +* Amazon S3 +* Backblaze B2 +* Box +* Ceph +* DigitalOcean Spaces +* Dreamhost +* Dropbox +* FTP +* Google Cloud Storage +* Google Drive +* HTTP +* Hubic +* Jottacloud +* IBM COS S3 +* Memset Memstore +* Mega +* Microsoft Azure Blob Storage +* Microsoft OneDrive +* Minio +* Nextcloud +* OVH +* OpenDrive +* Openstack Swift +* Oracle Cloud Storage +* ownCloud +* pCloud +* put.io +* QingStor +* Rackspace Cloud Files +* SFTP +* Wasabi +* WebDAV +* Yandex Disk +* The local filesystem + +Features + + * MD5/SHA1 hashes checked at all times for file integrity + * Timestamps preserved on files + * Partial syncs supported on a whole file basis + * [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files + * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical + * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality + * Can sync to and from network, eg two different cloud accounts + * Optional encryption ([Crypt](https://rclone.org/crypt/)) + * Optional cache ([Cache](https://rclone.org/cache/)) + * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/)) + +Links + + * [Home page](https://rclone.org/) + * [Github project page for source and bug tracker](https://github.com/ncw/rclone) + * [Rclone Forum](https://forum.rclone.org) + * Google+ page + * [Downloads](https://rclone.org/downloads/) + +# Install # + +Rclone is a Go program and comes as a single binary file. + +## Quickstart ## + + * [Download](https://rclone.org/downloads/) the relevant binary. + * Extract the `rclone` or `rclone.exe` binary from the archive + * Run `rclone config` to setup. See [rclone config docs](https://rclone.org/docs/) for more details. + +See below for some expanded Linux / macOS instructions. + +See the [Usage section](https://rclone.org/docs/) of the docs for how to use rclone, or +run `rclone -h`. + +## Script installation ## + +To install rclone on Linux/macOS/BSD systems, run: + + curl https://rclone.org/install.sh | sudo bash + +For beta installation, run: + + curl https://rclone.org/install.sh | sudo bash -s beta + +Note that this script checks the version of rclone installed first and +won't re-download if not needed. + +## Linux installation from precompiled binary ## + +Fetch and unpack + + curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip + unzip rclone-current-linux-amd64.zip + cd rclone-*-linux-amd64 + +Copy binary file + + sudo cp rclone /usr/bin/ + sudo chown root:root /usr/bin/rclone + sudo chmod 755 /usr/bin/rclone + +Install manpage + + sudo mkdir -p /usr/local/share/man/man1 + sudo cp rclone.1 /usr/local/share/man/man1/ + sudo mandb + +Run `rclone config` to setup. See [rclone config docs](https://rclone.org/docs/) for more details. + + rclone config + +## macOS installation from precompiled binary ## + +Download the latest version of rclone. + + cd && curl -O https://downloads.rclone.org/rclone-current-osx-amd64.zip + +Unzip the download and cd to the extracted folder. + + unzip -a rclone-current-osx-amd64.zip && cd rclone-*-osx-amd64 + +Move rclone to your $PATH. You will be prompted for your password. + + sudo mkdir -p /usr/local/bin + sudo mv rclone /usr/local/bin/ + +(the `mkdir` command is safe to run, even if the directory already exists). + +Remove the leftover files. + + cd .. && rm -rf rclone-*-osx-amd64 rclone-current-osx-amd64.zip + +Run `rclone config` to setup. See [rclone config docs](https://rclone.org/docs/) for more details. + + rclone config + +## Install from source ## + +Make sure you have at least [Go](https://golang.org/) 1.7 +installed. [Download go](https://golang.org/dl/) if necessary. The +latest release is recommended. Then + + git clone https://github.com/ncw/rclone.git + cd rclone + go build + ./rclone version + +You can also build and install rclone in the +[GOPATH](https://github.com/golang/go/wiki/GOPATH) (which defaults to +`~/go`) with: + + go get -u -v github.com/ncw/rclone + +and this will build the binary in `$GOPATH/bin` (`~/go/bin/rclone` by +default) after downloading the source to +`$GOPATH/src/github.com/ncw/rclone` (`~/go/src/github.com/ncw/rclone` +by default). + +## Installation with Ansible ## + +This can be done with [Stefan Weichinger's ansible +role](https://github.com/stefangweichinger/ansible-rclone). + +Instructions + + 1. `git clone https://github.com/stefangweichinger/ansible-rclone.git` into your local roles-directory + 2. add the role to the hosts you want rclone installed to: + +``` + - hosts: rclone-hosts + roles: + - rclone +``` + +Configure +--------- + +First, you'll need to configure rclone. As the object storage systems +have quite complicated authentication these are kept in a config file. +(See the `--config` entry for how to find the config file and choose +its location.) + +The easiest way to make the config is to run rclone with the config +option: + + rclone config + +See the following for detailed instructions for + + * [Alias](https://rclone.org/alias/) + * [Amazon Drive](https://rclone.org/amazonclouddrive/) + * [Amazon S3](https://rclone.org/s3/) + * [Backblaze B2](https://rclone.org/b2/) + * [Box](https://rclone.org/box/) + * [Cache](https://rclone.org/cache/) + * [Crypt](https://rclone.org/crypt/) - to encrypt other remotes + * [DigitalOcean Spaces](/s3/#digitalocean-spaces) + * [Dropbox](https://rclone.org/dropbox/) + * [FTP](https://rclone.org/ftp/) + * [Google Cloud Storage](https://rclone.org/googlecloudstorage/) + * [Google Drive](https://rclone.org/drive/) + * [HTTP](https://rclone.org/http/) + * [Hubic](https://rclone.org/hubic/) + * [Jottacloud](https://rclone.org/jottacloud/) + * [Mega](https://rclone.org/mega/) + * [Microsoft Azure Blob Storage](https://rclone.org/azureblob/) + * [Microsoft OneDrive](https://rclone.org/onedrive/) + * [Openstack Swift / Rackspace Cloudfiles / Memset Memstore](https://rclone.org/swift/) + * [OpenDrive](https://rclone.org/opendrive/) + * [Pcloud](https://rclone.org/pcloud/) + * [QingStor](https://rclone.org/qingstor/) + * [SFTP](https://rclone.org/sftp/) + * [WebDAV](https://rclone.org/webdav/) + * [Yandex Disk](https://rclone.org/yandex/) + * [The local filesystem](https://rclone.org/local/) + +Usage +----- + +Rclone syncs a directory tree from one storage system to another. + +Its syntax is like this + + Syntax: [options] subcommand + +Source and destination paths are specified by the name you gave the +storage system in the config file then the sub path, eg +"drive:myfolder" to look at "myfolder" in Google drive. + +You can define as many storage paths as you like in the config file. + +Subcommands +----------- + +rclone uses a system of subcommands. For example + + rclone ls remote:path # lists a re + rclone copy /local/path remote:path # copies /local/path to the remote + rclone sync /local/path remote:path # syncs /local/path to the remote + +## rclone config + +Enter an interactive configuration session. + +### Synopsis + +Enter an interactive configuration session where you can setup new +remotes and manage existing ones. You may also set or remove a +password to protect your configuration. + + +``` +rclone config [flags] +``` + +### Options + +``` + -h, --help help for config +``` + +## rclone copy + +Copy files from source to dest, skipping already copied + +### Synopsis + + +Copy the source to the destination. Doesn't transfer +unchanged files, testing by size and modification time or +MD5SUM. Doesn't delete files from the destination. + +Note that it is always the contents of the directory that is synced, +not the directory so when source:path is a directory, it's the +contents of source:path that are copied, not the directory name and +contents. + +If dest:path doesn't exist, it is created and the source:path contents +go there. + +For example + + rclone copy source:sourcepath dest:destpath + +Let's say there are two files in sourcepath + + sourcepath/one.txt + sourcepath/two.txt + +This copies them to + + destpath/one.txt + destpath/two.txt + +Not to + + destpath/sourcepath/one.txt + destpath/sourcepath/two.txt + +If you are familiar with `rsync`, rclone always works as if you had +written a trailing / - meaning "copy the contents of this directory". +This applies to all commands and whether you are talking about the +source or destination. + + +``` +rclone copy source:path dest:path [flags] +``` + +### Options + +``` + -h, --help help for copy +``` + +## rclone sync + +Make source and dest identical, modifying destination only. + +### Synopsis + + +Sync the source to the destination, changing the destination +only. Doesn't transfer unchanged files, testing by size and +modification time or MD5SUM. Destination is updated to match +source, including deleting files if necessary. + +**Important**: Since this can cause data loss, test first with the +`--dry-run` flag to see exactly what would be copied and deleted. + +Note that files in the destination won't be deleted if there were any +errors at any point. + +It is always the contents of the directory that is synced, not the +directory so when source:path is a directory, it's the contents of +source:path that are copied, not the directory name and contents. See +extended explanation in the `copy` command above if unsure. + +If dest:path doesn't exist, it is created and the source:path contents +go there. + + +``` +rclone sync source:path dest:path [flags] +``` + +### Options + +``` + -h, --help help for sync +``` + +## rclone move + +Move files from source to dest. + +### Synopsis + + +Moves the contents of the source directory to the destination +directory. Rclone will error if the source and destination overlap and +the remote does not support a server side directory move operation. + +If no filters are in use and if possible this will server side move +`source:path` into `dest:path`. After this `source:path` will no +longer longer exist. + +Otherwise for each file in `source:path` selected by the filters (if +any) this will move it into `dest:path`. If possible a server side +move will be used, otherwise it will copy it (server side if possible) +into `dest:path` then delete the original (if no errors on copy) in +`source:path`. + +If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag. + +**Important**: Since this can cause data loss, test first with the +--dry-run flag. + + +``` +rclone move source:path dest:path [flags] +``` + +### Options + +``` + --delete-empty-src-dirs Delete empty source dirs after move + -h, --help help for move +``` + +## rclone delete + +Remove the contents of path. + +### Synopsis + + +Remove the contents of path. Unlike `purge` it obeys include/exclude +filters so can be used to selectively delete files. + +Eg delete all files bigger than 100MBytes + +Check what would be deleted first (use either) + + rclone --min-size 100M lsl remote:path + rclone --dry-run --min-size 100M delete remote:path + +Then delete + + rclone --min-size 100M delete remote:path + +That reads "delete everything with a minimum size of 100 MB", hence +delete all files bigger than 100MBytes. + + +``` +rclone delete remote:path [flags] +``` + +### Options + +``` + -h, --help help for delete +``` + +## rclone purge + +Remove the path and all of its contents. + +### Synopsis + + +Remove the path and all of its contents. Note that this does not obey +include/exclude filters - everything will be removed. Use `delete` if +you want to selectively delete files. + + +``` +rclone purge remote:path [flags] +``` + +### Options + +``` + -h, --help help for purge +``` + +## rclone mkdir + +Make the path if it doesn't already exist. + +### Synopsis + +Make the path if it doesn't already exist. + +``` +rclone mkdir remote:path [flags] +``` + +### Options + +``` + -h, --help help for mkdir +``` + +## rclone rmdir + +Remove the path if empty. + +### Synopsis + + +Remove the path. Note that you can't remove a path with +objects in it, use purge for that. + +``` +rclone rmdir remote:path [flags] +``` + +### Options + +``` + -h, --help help for rmdir +``` + +## rclone check + +Checks the files in the source and destination match. + +### Synopsis + + +Checks the files in the source and destination match. It compares +sizes and hashes (MD5 or SHA1) and logs a report of files which don't +match. It doesn't alter the source or destination. + +If you supply the --size-only flag, it will only compare the sizes not +the hashes as well. Use this for a quick check. + +If you supply the --download flag, it will download the data from +both remotes and check them against each other on the fly. This can +be useful for remotes that don't support hashes or if you really want +to check all the data. + +If you supply the --one-way flag, it will only check that files in source +match the files in destination, not the other way around. Meaning extra files in +destination that are not in the source will not trigger an error. + + +``` +rclone check source:path dest:path [flags] +``` + +### Options + +``` + --download Check by downloading rather than with hash. + -h, --help help for check + --one-way Check one way only, source files must exist on remote +``` + +## rclone ls + +List the objects in the path with size and path. + +### Synopsis + + +Lists the objects in the source path to standard output in a human +readable format with size and path. Recurses by default. + +Eg + + $ rclone ls swift:bucket + 60295 bevajer5jef + 90613 canole + 94467 diwogej7 + 37600 fubuwic + + +Any of the filtering options can be applied to this commmand. + +There are several related list commands + + * `ls` to list size and path of objects only + * `lsl` to list modification time, size and path of objects only + * `lsd` to list directories only + * `lsf` to list objects and directories in easy to parse format + * `lsjson` to list objects and directories in JSON format + +`ls`,`lsl`,`lsd` are designed to be human readable. +`lsf` is designed to be human and machine readable. +`lsjson` is designed to be machine readable. + +Note that `ls` and `lsl` recurse by default - use "--max-depth 1" to stop the recursion. + +The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use "-R" to make them recurse. + +Listing a non existent directory will produce an error except for +remotes which can't have empty directories (eg s3, swift, gcs, etc - +the bucket based remotes). + + +``` +rclone ls remote:path [flags] +``` + +### Options + +``` + -h, --help help for ls +``` + +## rclone lsd + +List all directories/containers/buckets in the path. + +### Synopsis + + +Lists the directories in the source path to standard output. Does not +recurse by default. Use the -R flag to recurse. + +This command lists the total size of the directory (if known, -1 if +not), the modification time (if known, the current time if not), the +number of objects in the directory (if known, -1 if not) and the name +of the directory, Eg + + $ rclone lsd swift: + 494000 2018-04-26 08:43:20 10000 10000files + 65 2018-04-26 08:43:20 1 1File + +Or + + $ rclone lsd drive:test + -1 2016-10-17 17:41:53 -1 1000files + -1 2017-01-03 14:40:54 -1 2500files + -1 2017-07-08 14:39:28 -1 4000files + +If you just want the directory names use "rclone lsf --dirs-only". + + +Any of the filtering options can be applied to this commmand. + +There are several related list commands + + * `ls` to list size and path of objects only + * `lsl` to list modification time, size and path of objects only + * `lsd` to list directories only + * `lsf` to list objects and directories in easy to parse format + * `lsjson` to list objects and directories in JSON format + +`ls`,`lsl`,`lsd` are designed to be human readable. +`lsf` is designed to be human and machine readable. +`lsjson` is designed to be machine readable. + +Note that `ls` and `lsl` recurse by default - use "--max-depth 1" to stop the recursion. + +The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use "-R" to make them recurse. + +Listing a non existent directory will produce an error except for +remotes which can't have empty directories (eg s3, swift, gcs, etc - +the bucket based remotes). + + +``` +rclone lsd remote:path [flags] +``` + +### Options + +``` + -h, --help help for lsd + -R, --recursive Recurse into the listing. +``` + +## rclone lsl + +List the objects in path with modification time, size and path. + +### Synopsis + + +Lists the objects in the source path to standard output in a human +readable format with modification time, size and path. Recurses by default. + +Eg + + $ rclone lsl swift:bucket + 60295 2016-06-25 18:55:41.062626927 bevajer5jef + 90613 2016-06-25 18:55:43.302607074 canole + 94467 2016-06-25 18:55:43.046609333 diwogej7 + 37600 2016-06-25 18:55:40.814629136 fubuwic + + +Any of the filtering options can be applied to this commmand. + +There are several related list commands + + * `ls` to list size and path of objects only + * `lsl` to list modification time, size and path of objects only + * `lsd` to list directories only + * `lsf` to list objects and directories in easy to parse format + * `lsjson` to list objects and directories in JSON format + +`ls`,`lsl`,`lsd` are designed to be human readable. +`lsf` is designed to be human and machine readable. +`lsjson` is designed to be machine readable. + +Note that `ls` and `lsl` recurse by default - use "--max-depth 1" to stop the recursion. + +The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use "-R" to make them recurse. + +Listing a non existent directory will produce an error except for +remotes which can't have empty directories (eg s3, swift, gcs, etc - +the bucket based remotes). + + +``` +rclone lsl remote:path [flags] +``` + +### Options + +``` + -h, --help help for lsl +``` + +## rclone md5sum + +Produces an md5sum file for all the objects in the path. + +### Synopsis + + +Produces an md5sum file for all the objects in the path. This +is in the same format as the standard md5sum tool produces. + + +``` +rclone md5sum remote:path [flags] +``` + +### Options + +``` + -h, --help help for md5sum +``` + +## rclone sha1sum + +Produces an sha1sum file for all the objects in the path. + +### Synopsis + + +Produces an sha1sum file for all the objects in the path. This +is in the same format as the standard sha1sum tool produces. + + +``` +rclone sha1sum remote:path [flags] +``` + +### Options + +``` + -h, --help help for sha1sum +``` + +## rclone size + +Prints the total size and number of objects in remote:path. + +### Synopsis + +Prints the total size and number of objects in remote:path. + +``` +rclone size remote:path [flags] +``` + +### Options + +``` + -h, --help help for size + --json format output as JSON +``` + +## rclone version + +Show the version number. + +### Synopsis + + +Show the version number, the go version and the architecture. + +Eg + + $ rclone version + rclone v1.41 + - os/arch: linux/amd64 + - go version: go1.10 + +If you supply the --check flag, then it will do an online check to +compare your version with the latest release and the latest beta. + + $ rclone version --check + yours: 1.42.0.6 + latest: 1.42 (released 2018-06-16) + beta: 1.42.0.5 (released 2018-06-17) + +Or + + $ rclone version --check + yours: 1.41 + latest: 1.42 (released 2018-06-16) + upgrade: https://downloads.rclone.org/v1.42 + beta: 1.42.0.5 (released 2018-06-17) + upgrade: https://beta.rclone.org/v1.42-005-g56e1e820 + + + +``` +rclone version [flags] +``` + +### Options + +``` + --check Check for new version. + -h, --help help for version +``` + +## rclone cleanup + +Clean up the remote if possible + +### Synopsis + + +Clean up the remote if possible. Empty the trash or delete old file +versions. Not supported by all remotes. + + +``` +rclone cleanup remote:path [flags] +``` + +### Options + +``` + -h, --help help for cleanup +``` + +## rclone dedupe + +Interactively find duplicate files and delete/rename them. + +### Synopsis + + +By default `dedupe` interactively finds duplicate files and offers to +delete all but one or rename them to be different. Only useful with +Google Drive which can have duplicate file names. + +In the first pass it will merge directories with the same name. It +will do this iteratively until all the identical directories have been +merged. + +The `dedupe` command will delete all but one of any identical (same +md5sum) files it finds without confirmation. This means that for most +duplicated files the `dedupe` command will not be interactive. You +can use `--dry-run` to see what would happen without doing anything. + +Here is an example run. + +Before - with duplicates + + $ rclone lsl drive:dupes + 6048320 2016-03-05 16:23:16.798000000 one.txt + 6048320 2016-03-05 16:23:11.775000000 one.txt + 564374 2016-03-05 16:23:06.731000000 one.txt + 6048320 2016-03-05 16:18:26.092000000 one.txt + 6048320 2016-03-05 16:22:46.185000000 two.txt + 1744073 2016-03-05 16:22:38.104000000 two.txt + 564374 2016-03-05 16:22:52.118000000 two.txt + +Now the `dedupe` session + + $ rclone dedupe drive:dupes + 2016/03/05 16:24:37 Google drive root 'dupes': Looking for duplicates using interactive mode. + one.txt: Found 4 duplicates - deleting identical copies + one.txt: Deleting 2/3 identical duplicates (md5sum "1eedaa9fe86fd4b8632e2ac549403b36") + one.txt: 2 duplicates remain + 1: 6048320 bytes, 2016-03-05 16:23:16.798000000, md5sum 1eedaa9fe86fd4b8632e2ac549403b36 + 2: 564374 bytes, 2016-03-05 16:23:06.731000000, md5sum 7594e7dc9fc28f727c42ee3e0749de81 + s) Skip and do nothing + k) Keep just one (choose which in next step) + r) Rename all to be different (by changing file.jpg to file-1.jpg) + s/k/r> k + Enter the number of the file to keep> 1 + one.txt: Deleted 1 extra copies + two.txt: Found 3 duplicates - deleting identical copies + two.txt: 3 duplicates remain + 1: 564374 bytes, 2016-03-05 16:22:52.118000000, md5sum 7594e7dc9fc28f727c42ee3e0749de81 + 2: 6048320 bytes, 2016-03-05 16:22:46.185000000, md5sum 1eedaa9fe86fd4b8632e2ac549403b36 + 3: 1744073 bytes, 2016-03-05 16:22:38.104000000, md5sum 851957f7fb6f0bc4ce76be966d336802 + s) Skip and do nothing + k) Keep just one (choose which in next step) + r) Rename all to be different (by changing file.jpg to file-1.jpg) + s/k/r> r + two-1.txt: renamed from: two.txt + two-2.txt: renamed from: two.txt + two-3.txt: renamed from: two.txt + +The result being + + $ rclone lsl drive:dupes + 6048320 2016-03-05 16:23:16.798000000 one.txt + 564374 2016-03-05 16:22:52.118000000 two-1.txt + 6048320 2016-03-05 16:22:46.185000000 two-2.txt + 1744073 2016-03-05 16:22:38.104000000 two-3.txt + +Dedupe can be run non interactively using the `--dedupe-mode` flag or by using an extra parameter with the same value + + * `--dedupe-mode interactive` - interactive as above. + * `--dedupe-mode skip` - removes identical files then skips anything left. + * `--dedupe-mode first` - removes identical files then keeps the first one. + * `--dedupe-mode newest` - removes identical files then keeps the newest one. + * `--dedupe-mode oldest` - removes identical files then keeps the oldest one. + * `--dedupe-mode largest` - removes identical files then keeps the largest one. + * `--dedupe-mode rename` - removes identical files then renames the rest to be different. + +For example to rename all the identically named photos in your Google Photos directory, do + + rclone dedupe --dedupe-mode rename "drive:Google Photos" + +Or + + rclone dedupe rename "drive:Google Photos" + + +``` +rclone dedupe [mode] remote:path [flags] +``` + +### Options + +``` + --dedupe-mode string Dedupe mode interactive|skip|first|newest|oldest|rename. (default "interactive") + -h, --help help for dedupe +``` + +## rclone about + +Get quota information from the remote. + +### Synopsis + + +Get quota information from the remote, like bytes used/free/quota and bytes +used in the trash. Not supported by all remotes. + +This will print to stdout something like this: + + Total: 17G + Used: 7.444G + Free: 1.315G + Trashed: 100.000M + Other: 8.241G + +Where the fields are: + + * Total: total size available. + * Used: total size used + * Free: total amount this user could upload. + * Trashed: total amount in the trash + * Other: total amount in other storage (eg Gmail, Google Photos) + * Objects: total number of objects in the storage + +Note that not all the backends provide all the fields - they will be +missing if they are not known for that backend. Where it is known +that the value is unlimited the value will also be omitted. + +Use the --full flag to see the numbers written out in full, eg + + Total: 18253611008 + Used: 7993453766 + Free: 1411001220 + Trashed: 104857602 + Other: 8849156022 + +Use the --json flag for a computer readable output, eg + + { + "total": 18253611008, + "used": 7993453766, + "trashed": 104857602, + "other": 8849156022, + "free": 1411001220 + } + + +``` +rclone about remote: [flags] +``` + +### Options + +``` + --full Full numbers instead of SI units + -h, --help help for about + --json Format output as JSON +``` + +## rclone authorize + +Remote authorization. + +### Synopsis + + +Remote authorization. Used to authorize a remote or headless +rclone from a machine with a browser - use as instructed by +rclone config. + +``` +rclone authorize [flags] +``` + +### Options + +``` + -h, --help help for authorize +``` + +## rclone cachestats + +Print cache stats for a remote + +### Synopsis + + +Print cache stats for a remote in JSON format + + +``` +rclone cachestats source: [flags] +``` + +### Options + +``` + -h, --help help for cachestats +``` + +## rclone cat + +Concatenates any files and sends them to stdout. + +### Synopsis + + +rclone cat sends any files to standard output. + +You can use it like this to output a single file + + rclone cat remote:path/to/file + +Or like this to output any file in dir or subdirectories. + + rclone cat remote:path/to/dir + +Or like this to output any .txt files in dir or subdirectories. + + rclone --include "*.txt" cat remote:path/to/dir + +Use the --head flag to print characters only at the start, --tail for +the end and --offset and --count to print a section in the middle. +Note that if offset is negative it will count from the end, so +--offset -1 --count 1 is equivalent to --tail 1. + + +``` +rclone cat remote:path [flags] +``` + +### Options + +``` + --count int Only print N characters. (default -1) + --discard Discard the output instead of printing. + --head int Only print the first N characters. + -h, --help help for cat + --offset int Start printing at offset N (or from end if -ve). + --tail int Only print the last N characters. +``` + +## rclone config create + +Create a new remote with name, type and options. + +### Synopsis + + +Create a new remote of with and options. The options +should be passed in in pairs of . + +For example to make a swift remote of name myremote using auto config +you would do: + + rclone config create myremote swift env_auth true + + +``` +rclone config create [ ]* [flags] +``` + +### Options + +``` + -h, --help help for create +``` + +## rclone config delete + +Delete an existing remote . + +### Synopsis + +Delete an existing remote . + +``` +rclone config delete [flags] +``` + +### Options + +``` + -h, --help help for delete +``` + +## rclone config dump + +Dump the config file as JSON. + +### Synopsis + +Dump the config file as JSON. + +``` +rclone config dump [flags] +``` + +### Options + +``` + -h, --help help for dump +``` + +## rclone config edit + +Enter an interactive configuration session. + +### Synopsis + +Enter an interactive configuration session where you can setup new +remotes and manage existing ones. You may also set or remove a +password to protect your configuration. + + +``` +rclone config edit [flags] +``` + +### Options + +``` + -h, --help help for edit +``` + +## rclone config file + +Show path of configuration file in use. + +### Synopsis + +Show path of configuration file in use. + +``` +rclone config file [flags] +``` + +### Options + +``` + -h, --help help for file +``` + +## rclone config password + +Update password in an existing remote. + +### Synopsis + + +Update an existing remote's password. The password +should be passed in in pairs of . + +For example to set password of a remote of name myremote you would do: + + rclone config password myremote fieldname mypassword + + +``` +rclone config password [ ]+ [flags] +``` + +### Options + +``` + -h, --help help for password +``` + +## rclone config providers + +List in JSON format all the providers and options. + +### Synopsis + +List in JSON format all the providers and options. + +``` +rclone config providers [flags] +``` + +### Options + +``` + -h, --help help for providers +``` + +## rclone config show + +Print (decrypted) config file, or the config for a single remote. + +### Synopsis + +Print (decrypted) config file, or the config for a single remote. + +``` +rclone config show [] [flags] +``` + +### Options + +``` + -h, --help help for show +``` + +## rclone config update + +Update options in an existing remote. + +### Synopsis + + +Update an existing remote's options. The options should be passed in +in pairs of . + +For example to update the env_auth field of a remote of name myremote you would do: + + rclone config update myremote swift env_auth true + + +``` +rclone config update [ ]+ [flags] +``` + +### Options + +``` + -h, --help help for update +``` + +## rclone copyto + +Copy files from source to dest, skipping already copied + +### Synopsis + + +If source:path is a file or directory then it copies it to a file or +directory named dest:path. + +This can be used to upload single files to other than their current +name. If the source is a directory then it acts exactly like the copy +command. + +So + + rclone copyto src dst + +where src and dst are rclone paths, either remote:path or +/path/to/local or C:\windows\path\if\on\windows. + +This will: + + if src is file + copy it to dst, overwriting an existing file if it exists + if src is directory + copy it to dst, overwriting existing files if they exist + see copy command for full details + +This doesn't transfer unchanged files, testing by size and +modification time or MD5SUM. It doesn't delete files from the +destination. + + +``` +rclone copyto source:path dest:path [flags] +``` + +### Options + +``` + -h, --help help for copyto +``` + +## rclone copyurl + +Copy url content to dest. + +### Synopsis + + +Download urls content and copy it to destination +without saving it in tmp storage. + + +``` +rclone copyurl https://example.com dest:path [flags] +``` + +### Options + +``` + -h, --help help for copyurl +``` + +## rclone cryptcheck + +Cryptcheck checks the integrity of a crypted remote. + +### Synopsis + + +rclone cryptcheck checks a remote against a crypted remote. This is +the equivalent of running rclone check, but able to check the +checksums of the crypted remote. + +For it to work the underlying remote of the cryptedremote must support +some kind of checksum. + +It works by reading the nonce from each file on the cryptedremote: and +using that to encrypt each file on the remote:. It then checks the +checksum of the underlying file on the cryptedremote: against the +checksum of the file it has just encrypted. + +Use it like this + + rclone cryptcheck /path/to/files encryptedremote:path + +You can use it like this also, but that will involve downloading all +the files in remote:path. + + rclone cryptcheck remote:path encryptedremote:path + +After it has run it will log the status of the encryptedremote:. + +If you supply the --one-way flag, it will only check that files in source +match the files in destination, not the other way around. Meaning extra files in +destination that are not in the source will not trigger an error. + + +``` +rclone cryptcheck remote:path cryptedremote:path [flags] +``` + +### Options + +``` + -h, --help help for cryptcheck + --one-way Check one way only, source files must exist on destination +``` + +## rclone cryptdecode + +Cryptdecode returns unencrypted file names. + +### Synopsis + + +rclone cryptdecode returns unencrypted file names when provided with +a list of encrypted file names. List limit is 10 items. + +If you supply the --reverse flag, it will return encrypted file names. + +use it like this + + rclone cryptdecode encryptedremote: encryptedfilename1 encryptedfilename2 + + rclone cryptdecode --reverse encryptedremote: filename1 filename2 + + +``` +rclone cryptdecode encryptedremote: encryptedfilename [flags] +``` + +### Options + +``` + -h, --help help for cryptdecode + --reverse Reverse cryptdecode, encrypts filenames +``` + +## rclone dbhashsum + +Produces a Dropbox hash file for all the objects in the path. + +### Synopsis + + +Produces a Dropbox hash file for all the objects in the path. The +hashes are calculated according to [Dropbox content hash +rules](https://www.dropbox.com/developers/reference/content-hash). +The output is in the same format as md5sum and sha1sum. + + +``` +rclone dbhashsum remote:path [flags] +``` + +### Options + +``` + -h, --help help for dbhashsum +``` + +## rclone deletefile + +Remove a single file from remote. + +### Synopsis + + +Remove a single file from remote. Unlike `delete` it cannot be used to +remove a directory and it doesn't obey include/exclude filters - if the specified file exists, +it will always be removed. + + +``` +rclone deletefile remote:path [flags] +``` + +### Options + +``` + -h, --help help for deletefile +``` + +## rclone genautocomplete + +Output completion script for a given shell. + +### Synopsis + + +Generates a shell completion script for rclone. +Run with --help to list the supported shells. + + +### Options + +``` + -h, --help help for genautocomplete +``` + +## rclone genautocomplete bash + +Output bash completion script for rclone. + +### Synopsis + + +Generates a bash shell autocompletion script for rclone. + +This writes to /etc/bash_completion.d/rclone by default so will +probably need to be run with sudo or as root, eg + + sudo rclone genautocomplete bash + +Logout and login again to use the autocompletion scripts, or source +them directly + + . /etc/bash_completion + +If you supply a command line argument the script will be written +there. + + +``` +rclone genautocomplete bash [output_file] [flags] +``` + +### Options + +``` + -h, --help help for bash +``` + +## rclone genautocomplete zsh + +Output zsh completion script for rclone. + +### Synopsis + + +Generates a zsh autocompletion script for rclone. + +This writes to /usr/share/zsh/vendor-completions/_rclone by default so will +probably need to be run with sudo or as root, eg + + sudo rclone genautocomplete zsh + +Logout and login again to use the autocompletion scripts, or source +them directly + + autoload -U compinit && compinit + +If you supply a command line argument the script will be written +there. + + +``` +rclone genautocomplete zsh [output_file] [flags] +``` + +### Options + +``` + -h, --help help for zsh +``` + +## rclone gendocs + +Output markdown docs for rclone to the directory supplied. + +### Synopsis + + +This produces markdown docs for the rclone commands to the directory +supplied. These are in a format suitable for hugo to render into the +rclone.org website. + +``` +rclone gendocs output_directory [flags] +``` + +### Options + +``` + -h, --help help for gendocs +``` + +## rclone hashsum + +Produces an hashsum file for all the objects in the path. + +### Synopsis + + +Produces a hash file for all the objects in the path using the hash +named. The output is in the same format as the standard +md5sum/sha1sum tool. + +Run without a hash to see the list of supported hashes, eg + + $ rclone hashsum + Supported hashes are: + * MD5 + * SHA-1 + * DropboxHash + * QuickXorHash + +Then + + $ rclone hashsum MD5 remote:path + + +``` +rclone hashsum remote:path [flags] +``` + +### Options + +``` + -h, --help help for hashsum +``` + +## rclone link + +Generate public link to file/folder. + +### Synopsis + + +rclone link will create or retrieve a public link to the given file or folder. + + rclone link remote:path/to/file + rclone link remote:path/to/folder/ + +If successful, the last line of the output will contain the link. Exact +capabilities depend on the remote, but the link will always be created with +the least constraints – e.g. no expiry, no password protection, accessible +without account. + + +``` +rclone link remote:path [flags] +``` + +### Options + +``` + -h, --help help for link +``` + +## rclone listremotes + +List all the remotes in the config file. + +### Synopsis + + +rclone listremotes lists all the available remotes from the config file. + +When uses with the -l flag it lists the types too. + + +``` +rclone listremotes [flags] +``` + +### Options + +``` + -h, --help help for listremotes + -l, --long Show the type as well as names. +``` + +## rclone lsf + +List directories and objects in remote:path formatted for parsing + +### Synopsis + + +List the contents of the source path (directories and objects) to +standard output in a form which is easy to parse by scripts. By +default this will just be the names of the objects and directories, +one per line. The directories will have a / suffix. + +Eg + + $ rclone lsf swift:bucket + bevajer5jef + canole + diwogej7 + ferejej3gux/ + fubuwic + +Use the --format option to control what gets listed. By default this +is just the path, but you can use these parameters to control the +output: + + p - path + s - size + t - modification time + h - hash + i - ID of object if known + m - MimeType of object if known + +So if you wanted the path, size and modification time, you would use +--format "pst", or maybe --format "tsp" to put the path last. + +Eg + + $ rclone lsf --format "tsp" swift:bucket + 2016-06-25 18:55:41;60295;bevajer5jef + 2016-06-25 18:55:43;90613;canole + 2016-06-25 18:55:43;94467;diwogej7 + 2018-04-26 08:50:45;0;ferejej3gux/ + 2016-06-25 18:55:40;37600;fubuwic + +If you specify "h" in the format you will get the MD5 hash by default, +use the "--hash" flag to change which hash you want. Note that this +can be returned as an empty string if it isn't available on the object +(and for directories), "ERROR" if there was an error reading it from +the object and "UNSUPPORTED" if that object does not support that hash +type. + +For example to emulate the md5sum command you can use + + rclone lsf -R --hash MD5 --format hp --separator " " --files-only . + +Eg + + $ rclone lsf -R --hash MD5 --format hp --separator " " --files-only swift:bucket + 7908e352297f0f530b84a756f188baa3 bevajer5jef + cd65ac234e6fea5925974a51cdd865cc canole + 03b5341b4f234b9d984d03ad076bae91 diwogej7 + 8fd37c3810dd660778137ac3a66cc06d fubuwic + 99713e14a4c4ff553acaf1930fad985b gixacuh7ku + +(Though "rclone md5sum ." is an easier way of typing this.) + +By default the separator is ";" this can be changed with the +--separator flag. Note that separators aren't escaped in the path so +putting it last is a good strategy. + +Eg + + $ rclone lsf --separator "," --format "tshp" swift:bucket + 2016-06-25 18:55:41,60295,7908e352297f0f530b84a756f188baa3,bevajer5jef + 2016-06-25 18:55:43,90613,cd65ac234e6fea5925974a51cdd865cc,canole + 2016-06-25 18:55:43,94467,03b5341b4f234b9d984d03ad076bae91,diwogej7 + 2018-04-26 08:52:53,0,,ferejej3gux/ + 2016-06-25 18:55:40,37600,8fd37c3810dd660778137ac3a66cc06d,fubuwic + +You can output in CSV standard format. This will escape things in " +if they contain , + +Eg + + $ rclone lsf --csv --files-only --format ps remote:path + test.log,22355 + test.sh,449 + "this file contains a comma, in the file name.txt",6 + +Note that the --absolute parameter is useful for making lists of files +to pass to an rclone copy with the --files-from flag. + +For example to find all the files modified within one day and copy +those only (without traversing the whole directory structure): + + rclone lsf --absolute --files-only --max-age 1d /path/to/local > new_files + rclone copy --files-from new_files /path/to/local remote:path + + +Any of the filtering options can be applied to this commmand. + +There are several related list commands + + * `ls` to list size and path of objects only + * `lsl` to list modification time, size and path of objects only + * `lsd` to list directories only + * `lsf` to list objects and directories in easy to parse format + * `lsjson` to list objects and directories in JSON format + +`ls`,`lsl`,`lsd` are designed to be human readable. +`lsf` is designed to be human and machine readable. +`lsjson` is designed to be machine readable. + +Note that `ls` and `lsl` recurse by default - use "--max-depth 1" to stop the recursion. + +The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use "-R" to make them recurse. + +Listing a non existent directory will produce an error except for +remotes which can't have empty directories (eg s3, swift, gcs, etc - +the bucket based remotes). + + +``` +rclone lsf remote:path [flags] +``` + +### Options + +``` + --absolute Put a leading / in front of path names. + --csv Output in CSV format. + -d, --dir-slash Append a slash to directory names. (default true) + --dirs-only Only list directories. + --files-only Only list files. + -F, --format string Output format - see help for details (default "p") + --hash h Use this hash when h is used in the format MD5|SHA-1|DropboxHash (default "MD5") + -h, --help help for lsf + -R, --recursive Recurse into the listing. + -s, --separator string Separator for the items in the format. (default ";") +``` + +## rclone lsjson + +List directories and objects in the path in JSON format. + +### Synopsis + +List directories and objects in the path in JSON format. + +The output is an array of Items, where each Item looks like this + + { + "Hashes" : { + "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f", + "MD5" : "b1946ac92492d2347c6235b4d2611184", + "DropboxHash" : "ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc" + }, + "ID": "y2djkhiujf83u33", + "OrigID": "UYOJVTUW00Q1RzTDA", + "IsDir" : false, + "MimeType" : "application/octet-stream", + "ModTime" : "2017-05-31T16:15:57.034468261+01:00", + "Name" : "file.txt", + "Encrypted" : "v0qpsdq8anpci8n929v3uu9338", + "Path" : "full/path/goes/here/file.txt", + "Size" : 6 + } + +If --hash is not specified the Hashes property won't be emitted. + +If --no-modtime is specified then ModTime will be blank. + +If --encrypted is not specified the Encrypted won't be emitted. + +The Path field will only show folders below the remote path being listed. +If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt" +will be "subfolder/file.txt", not "remote:path/subfolder/file.txt". +When used without --recursive the Path will always be the same as Name. + +The time is in RFC3339 format with nanosecond precision. + +The whole output can be processed as a JSON blob, or alternatively it +can be processed line by line as each item is written one to a line. + +Any of the filtering options can be applied to this commmand. + +There are several related list commands + + * `ls` to list size and path of objects only + * `lsl` to list modification time, size and path of objects only + * `lsd` to list directories only + * `lsf` to list objects and directories in easy to parse format + * `lsjson` to list objects and directories in JSON format + +`ls`,`lsl`,`lsd` are designed to be human readable. +`lsf` is designed to be human and machine readable. +`lsjson` is designed to be machine readable. + +Note that `ls` and `lsl` recurse by default - use "--max-depth 1" to stop the recursion. + +The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use "-R" to make them recurse. + +Listing a non existent directory will produce an error except for +remotes which can't have empty directories (eg s3, swift, gcs, etc - +the bucket based remotes). + + +``` +rclone lsjson remote:path [flags] +``` + +### Options + +``` + -M, --encrypted Show the encrypted names. + --hash Include hashes in the output (may take longer). + -h, --help help for lsjson + --no-modtime Don't read the modification time (can speed things up). + --original Show the ID of the underlying Object. + -R, --recursive Recurse into the listing. +``` + +## rclone mount + +Mount the remote as a mountpoint. **EXPERIMENTAL** + +### Synopsis + + +rclone mount allows Linux, FreeBSD, macOS and Windows to +mount any of Rclone's cloud storage systems as a file system with +FUSE. + +This is **EXPERIMENTAL** - use with care. + +First set up your remote using `rclone config`. Check it works with `rclone ls` etc. + +Start the mount like this + + rclone mount remote:path/to/files /path/to/local/mount + +Or on Windows like this where X: is an unused drive letter + + rclone mount remote:path/to/files X: + +When the program ends, either via Ctrl+C or receiving a SIGINT or SIGTERM signal, +the mount is automatically stopped. + +The umount operation can fail, for example when the mountpoint is busy. +When that happens, it is the user's responsibility to stop the mount manually with + + # Linux + fusermount -u /path/to/local/mount + # OS X + umount /path/to/local/mount + +### Installing on Windows + +To run rclone mount on Windows, you will need to +download and install [WinFsp](http://www.secfs.net/winfsp/). + +WinFsp is an [open source](https://github.com/billziss-gh/winfsp) +Windows File System Proxy which makes it easy to write user space file +systems for Windows. It provides a FUSE emulation layer which rclone +uses combination with +[cgofuse](https://github.com/billziss-gh/cgofuse). Both of these +packages are by Bill Zissimopoulos who was very helpful during the +implementation of rclone mount for Windows. + +#### Windows caveats + +Note that drives created as Administrator are not visible by other +accounts (including the account that was elevated as +Administrator). So if you start a Windows drive from an Administrative +Command Prompt and then try to access the same drive from Explorer +(which does not run as Administrator), you will not be able to see the +new drive. + +The easiest way around this is to start the drive from a normal +command prompt. It is also possible to start a drive from the SYSTEM +account (using [the WinFsp.Launcher +infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)) +which creates drives accessible for everyone on the system or +alternatively using [the nssm service manager](https://nssm.cc/usage). + +### Limitations + +Without the use of "--vfs-cache-mode" this can only write files +sequentially, it can only seek when reading. This means that many +applications won't work with their files on an rclone mount without +"--vfs-cache-mode writes" or "--vfs-cache-mode full". See the [File +Caching](#file-caching) section for more info. + +The bucket based remotes (eg Swift, S3, Google Compute Storage, B2, +Hubic) won't work from the root - you will need to specify a bucket, +or a path within the bucket. So `swift:` won't work whereas +`swift:bucket` will as will `swift:bucket/path`. +None of these support the concept of directories, so empty +directories will have a tendency to disappear once they fall out of +the directory cache. + +Only supported on Linux, FreeBSD, OS X and Windows at the moment. + +### rclone mount vs rclone sync/copy + +File systems expect things to be 100% reliable, whereas cloud storage +systems are a long way from 100% reliable. The rclone sync/copy +commands cope with this with lots of retries. However rclone mount +can't use retries in the same way without making local copies of the +uploads. Look at the **EXPERIMENTAL** [file caching](#file-caching) +for solutions to make mount mount more reliable. + +### Attribute caching + +You can use the flag --attr-timeout to set the time the kernel caches +the attributes (size, modification time etc) for directory entries. + +The default is "1s" which caches files just long enough to avoid +too many callbacks to rclone from the kernel. + +In theory 0s should be the correct value for filesystems which can +change outside the control of the kernel. However this causes quite a +few problems such as +[rclone using too much memory](https://github.com/ncw/rclone/issues/2157), +[rclone not serving files to samba](https://forum.rclone.org/t/rclone-1-39-vs-1-40-mount-issue/5112) +and [excessive time listing directories](https://github.com/ncw/rclone/issues/2095#issuecomment-371141147). + +The kernel can cache the info about a file for the time given by +"--attr-timeout". You may see corruption if the remote file changes +length during this window. It will show up as either a truncated file +or a file with garbage on the end. With "--attr-timeout 1s" this is +very unlikely but not impossible. The higher you set "--attr-timeout" +the more likely it is. The default setting of "1s" is the lowest +setting which mitigates the problems above. + +If you set it higher ('10s' or '1m' say) then the kernel will call +back to rclone less often making it more efficient, however there is +more chance of the corruption issue above. + +If files don't change on the remote outside of the control of rclone +then there is no chance of corruption. + +This is the same as setting the attr_timeout option in mount.fuse. + +### Filters + +Note that all the rclone filters can be used to select a subset of the +files to be visible in the mount. + +### systemd + +When running rclone mount as a systemd service, it is possible +to use Type=notify. In this case the service will enter the started state +after the mountpoint has been successfully set up. +Units having the rclone mount service specified as a requirement +will see all files and folders immediately in this mode. + +### chunked reading ### + +--vfs-read-chunk-size will enable reading the source objects in parts. +This can reduce the used download quota for some remotes by requesting only chunks +from the remote that are actually read at the cost of an increased number of requests. + +When --vfs-read-chunk-size-limit is also specified and greater than --vfs-read-chunk-size, +the chunk size for each open file will get doubled for each chunk read, until the +specified value is reached. A value of -1 will disable the limit and the chunk size will +grow indefinitely. + +With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the following +parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on. +When --vfs-read-chunk-size-limit 500M is specified, the result would be +0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on. + +Chunked reading will only work with --vfs-cache-mode < full, as the file will always +be copied to the vfs cache before opening with --vfs-cache-mode full. + +### Directory Cache + +Using the `--dir-cache-time` flag, you can set how long a +directory should be considered up to date and not refreshed from the +backend. Changes made locally in the mount may appear immediately or +invalidate the cache. However, changes done on the remote will only +be picked up once the cache expires. + +Alternatively, you can send a `SIGHUP` signal to rclone for +it to flush all directory caches, regardless of how old they are. +Assuming only one rclone instance is running, you can reset the cache +like this: + + kill -SIGHUP $(pidof rclone) + +If you configure rclone with a [remote control](/rc) then you can use +rclone rc to flush the whole directory cache: + + rclone rc vfs/forget + +Or individual files or directories: + + rclone rc vfs/forget file=path/to/file dir=path/to/dir + +### File Buffering + +The `--buffer-size` flag determines the amount of memory, +that will be used to buffer data in advance. + +Each open file descriptor will try to keep the specified amount of +data in memory at all times. The buffered data is bound to one file +descriptor and won't be shared between multiple open file descriptors +of the same file. + +This flag is a upper limit for the used memory per file descriptor. +The buffer will only use memory for data that is downloaded but not +not yet read. If the buffer is empty, only a small amount of memory +will be used. +The maximum memory used by rclone for buffering can be up to +`--buffer-size * open files`. + +### File Caching + +**NB** File caching is **EXPERIMENTAL** - use with care! + +These flags control the VFS file caching options. The VFS layer is +used by rclone mount to make a cloud storage system work more like a +normal file system. + +You'll need to enable VFS caching if you want, for example, to read +and write simultaneously to a file. See below for more details. + +Note that the VFS cache works in addition to the cache backend and you +may find that you need one or the other or both. + + --cache-dir string Directory rclone will use for caching. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + +If run with `-vv` rclone will print the location of the file cache. The +files are stored in the user cache file area which is OS dependent but +can be controlled with `--cache-dir` or setting the appropriate +environment variable. + +The cache has 4 different modes selected by `--vfs-cache-mode`. +The higher the cache mode the more compatible rclone becomes at the +cost of using disk space. + +Note that files are written back to the remote only when they are +closed so if rclone is quit or dies with open files then these won't +get written back to the remote. However they will still be in the on +disk cache. + +#### --vfs-cache-mode off + +In this mode the cache will read directly from the remote and write +directly to the remote without caching anything on disk. + +This will mean some operations are not possible + + * Files can't be opened for both read AND write + * Files opened for write can't be seeked + * Existing files opened for write must have O_TRUNC set + * Files open for read with O_TRUNC will be opened write only + * Files open for write only will behave as if O_TRUNC was supplied + * Open modes O_APPEND, O_TRUNC are ignored + * If an upload fails it can't be retried + +#### --vfs-cache-mode minimal + +This is very similar to "off" except that files opened for read AND +write will be buffered to disks. This means that files opened for +write will be a lot more compatible, but uses the minimal disk space. + +These operations are not possible + + * Files opened for write only can't be seeked + * Existing files opened for write must have O_TRUNC set + * Files opened for write only will ignore O_APPEND, O_TRUNC + * If an upload fails it can't be retried + +#### --vfs-cache-mode writes + +In this mode files opened for read only are still read directly from +the remote, write only and read/write files are buffered to disk +first. + +This mode should support all normal file system operations. + +If an upload fails it will be retried up to --low-level-retries times. + +#### --vfs-cache-mode full + +In this mode all reads and writes are buffered to and from disk. When +a file is opened for read it will be downloaded in its entirety first. + +This may be appropriate for your needs, or you may prefer to look at +the cache backend which does a much more sophisticated job of caching, +including caching directory hierarchies and chunks of files. + +In this mode, unlike the others, when a file is written to the disk, +it will be kept on the disk after it is written to the remote. It +will be purged on a schedule according to `--vfs-cache-max-age`. + +This mode should support all normal file system operations. + +If an upload or download fails it will be retried up to +--low-level-retries times. + + +``` +rclone mount remote:path /path/to/mountpoint [flags] +``` + +### Options + +``` + --allow-non-empty Allow mounting over a non-empty directory. + --allow-other Allow access to other users. + --allow-root Allow access to root user. + --attr-timeout duration Time for which file/directory attributes are cached. (default 1s) + --daemon Run mount as a daemon (background mode). + --daemon-timeout duration Time limit for rclone to respond to kernel (not supported by all OSes). + --debug-fuse Debug the FUSE internals - needs -v. + --default-permissions Makes kernel enforce access control based on the file mode. + --dir-cache-time duration Time to cache directory entries for. (default 5m0s) + --fuse-flag stringArray Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required. + --gid uint32 Override the gid field set by the filesystem. (default 502) + -h, --help help for mount + --max-read-ahead int The number of bytes that can be prefetched for sequential reads. (default 128k) + --no-checksum Don't compare checksums on up/download. + --no-modtime Don't read/write the modification time (can speed things up). + --no-seek Don't allow seeking in files. + -o, --option stringArray Option for libfuse/WinFsp. Repeat if required. + --poll-interval duration Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable. (default 1m0s) + --read-only Mount read-only. + --uid uint32 Override the uid field set by the filesystem. (default 502) + --umask int Override the permission bits set by the filesystem. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + --vfs-read-chunk-size int Read the source objects in chunks. (default 128M) + --vfs-read-chunk-size-limit int If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off) + --volname string Set the volume name (not supported by all OSes). + --write-back-cache Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used. +``` + +## rclone moveto + +Move file or directory from source to dest. + +### Synopsis + + +If source:path is a file or directory then it moves it to a file or +directory named dest:path. + +This can be used to rename files or upload single files to other than +their existing name. If the source is a directory then it acts exacty +like the move command. + +So + + rclone moveto src dst + +where src and dst are rclone paths, either remote:path or +/path/to/local or C:\windows\path\if\on\windows. + +This will: + + if src is file + move it to dst, overwriting an existing file if it exists + if src is directory + move it to dst, overwriting existing files if they exist + see move command for full details + +This doesn't transfer unchanged files, testing by size and +modification time or MD5SUM. src will be deleted on successful +transfer. + +**Important**: Since this can cause data loss, test first with the +--dry-run flag. + + +``` +rclone moveto source:path dest:path [flags] +``` + +### Options + +``` + -h, --help help for moveto +``` + +## rclone ncdu + +Explore a remote with a text based user interface. + +### Synopsis + + +This displays a text based user interface allowing the navigation of a +remote. It is most useful for answering the question - "What is using +all my disk space?". + + + +To make the user interface it first scans the entire remote given and +builds an in memory representation. rclone ncdu can be used during +this scanning phase and you will see it building up the directory +structure as it goes along. + +Here are the keys - press '?' to toggle the help on and off + + ↑,↓ or k,j to Move + →,l to enter + ←,h to return + c toggle counts + g toggle graph + n,s,C sort by name,size,count + ^L refresh screen + ? to toggle help on and off + q/ESC/c-C to quit + +This an homage to the [ncdu tool](https://dev.yorhel.nl/ncdu) but for +rclone remotes. It is missing lots of features at the moment, most +importantly deleting files, but is useful as it stands. + + +``` +rclone ncdu remote:path [flags] +``` + +### Options + +``` + -h, --help help for ncdu +``` + +## rclone obscure + +Obscure password for use in the rclone.conf + +### Synopsis + +Obscure password for use in the rclone.conf + +``` +rclone obscure password [flags] +``` + +### Options + +``` + -h, --help help for obscure +``` + +## rclone rc + +Run a command against a running rclone. + +### Synopsis + + +This runs a command against a running rclone. By default it will use +that specified in the --rc-addr command. + +Arguments should be passed in as parameter=value. + +The result will be returned as a JSON object by default. + +Use "rclone rc" to see a list of all possible commands. + +``` +rclone rc commands parameter [flags] +``` + +### Options + +``` + -h, --help help for rc + --no-output If set don't output the JSON result. + --url string URL to connect to rclone remote control. (default "http://localhost:5572/") +``` + +## rclone rcat + +Copies standard input to file on remote. + +### Synopsis + + +rclone rcat reads from standard input (stdin) and copies it to a +single remote file. + + echo "hello world" | rclone rcat remote:path/to/file + ffmpeg - | rclone rcat remote:path/to/file + +If the remote file already exists, it will be overwritten. + +rcat will try to upload small files in a single request, which is +usually more efficient than the streaming/chunked upload endpoints, +which use multiple requests. Exact behaviour depends on the remote. +What is considered a small file may be set through +`--streaming-upload-cutoff`. Uploading only starts after +the cutoff is reached or if the file ends before that. The data +must fit into RAM. The cutoff needs to be small enough to adhere +the limits of your remote, please see there. Generally speaking, +setting this cutoff too high will decrease your performance. + +Note that the upload can also not be retried because the data is +not kept around until the upload succeeds. If you need to transfer +a lot of data, you're better off caching locally and then +`rclone move` it to the destination. + +``` +rclone rcat remote:path [flags] +``` + +### Options + +``` + -h, --help help for rcat +``` + +## rclone rmdirs + +Remove empty directories under the path. + +### Synopsis + +This removes any empty directories (or directories that only contain +empty directories) under the path that it finds, including the path if +it has nothing in. + +If you supply the --leave-root flag, it will not remove the root directory. + +This is useful for tidying up remotes that rclone has left a lot of +empty directories in. + + + +``` +rclone rmdirs remote:path [flags] +``` + +### Options + +``` + -h, --help help for rmdirs + --leave-root Do not remove root directory if empty +``` + +## rclone serve + +Serve a remote over a protocol. + +### Synopsis + +rclone serve is used to serve a remote over a given protocol. This +command requires the use of a subcommand to specify the protocol, eg + + rclone serve http remote: + +Each subcommand has its own options which you can see in their help. + + +``` +rclone serve [opts] [flags] +``` + +### Options + +``` + -h, --help help for serve +``` + +## rclone serve http + +Serve the remote over HTTP. + +### Synopsis + +rclone serve http implements a basic web server to serve the remote +over HTTP. This can be viewed in a web browser or you can make a +remote of type http read from it. + +You can use the filter flags (eg --include, --exclude) to control what +is served. + +The server will log errors. Use -v to see access logs. + +--bwlimit will be respected for file transfers. Use --stats to +control the stats printing. + +### Server options + +Use --addr to specify which IP address and port the server should +listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all +IPs. By default it only listens on localhost. You can use port +:0 to let the OS choose an available port. + +If you set --addr to listen on a public or LAN accessible IP address +then using Authentication is advised - see the next section for info. + +--server-read-timeout and --server-write-timeout can be used to +control the timeouts on the server. Note that this is the total time +for a transfer. + +--max-header-bytes controls the maximum number of bytes the server will +accept in the HTTP header. + +#### Authentication + +By default this will serve files without needing a login. + +You can either use an htpasswd file which can take lots of users, or +set a single username and password with the --user and --pass flags. + +Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is +in standard apache format and supports MD5, SHA1 and BCrypt for basic +authentication. Bcrypt is recommended. + +To create an htpasswd file: + + touch htpasswd + htpasswd -B htpasswd user + htpasswd -B htpasswd anotherUser + +The password file can be updated while rclone is running. + +Use --realm to set the authentication realm. + +#### SSL/TLS + +By default this will serve over http. If you want you can serve over +https. You will need to supply the --cert and --key flags. If you +wish to do client side certificate validation then you will need to +supply --client-ca also. + +--cert should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. --key should be the PEM encoded +private key and --client-ca should be the PEM encoded client +certificate authority certificate. + +### Directory Cache + +Using the `--dir-cache-time` flag, you can set how long a +directory should be considered up to date and not refreshed from the +backend. Changes made locally in the mount may appear immediately or +invalidate the cache. However, changes done on the remote will only +be picked up once the cache expires. + +Alternatively, you can send a `SIGHUP` signal to rclone for +it to flush all directory caches, regardless of how old they are. +Assuming only one rclone instance is running, you can reset the cache +like this: + + kill -SIGHUP $(pidof rclone) + +If you configure rclone with a [remote control](/rc) then you can use +rclone rc to flush the whole directory cache: + + rclone rc vfs/forget + +Or individual files or directories: + + rclone rc vfs/forget file=path/to/file dir=path/to/dir + +### File Buffering + +The `--buffer-size` flag determines the amount of memory, +that will be used to buffer data in advance. + +Each open file descriptor will try to keep the specified amount of +data in memory at all times. The buffered data is bound to one file +descriptor and won't be shared between multiple open file descriptors +of the same file. + +This flag is a upper limit for the used memory per file descriptor. +The buffer will only use memory for data that is downloaded but not +not yet read. If the buffer is empty, only a small amount of memory +will be used. +The maximum memory used by rclone for buffering can be up to +`--buffer-size * open files`. + +### File Caching + +**NB** File caching is **EXPERIMENTAL** - use with care! + +These flags control the VFS file caching options. The VFS layer is +used by rclone mount to make a cloud storage system work more like a +normal file system. + +You'll need to enable VFS caching if you want, for example, to read +and write simultaneously to a file. See below for more details. + +Note that the VFS cache works in addition to the cache backend and you +may find that you need one or the other or both. + + --cache-dir string Directory rclone will use for caching. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + +If run with `-vv` rclone will print the location of the file cache. The +files are stored in the user cache file area which is OS dependent but +can be controlled with `--cache-dir` or setting the appropriate +environment variable. + +The cache has 4 different modes selected by `--vfs-cache-mode`. +The higher the cache mode the more compatible rclone becomes at the +cost of using disk space. + +Note that files are written back to the remote only when they are +closed so if rclone is quit or dies with open files then these won't +get written back to the remote. However they will still be in the on +disk cache. + +#### --vfs-cache-mode off + +In this mode the cache will read directly from the remote and write +directly to the remote without caching anything on disk. + +This will mean some operations are not possible + + * Files can't be opened for both read AND write + * Files opened for write can't be seeked + * Existing files opened for write must have O_TRUNC set + * Files open for read with O_TRUNC will be opened write only + * Files open for write only will behave as if O_TRUNC was supplied + * Open modes O_APPEND, O_TRUNC are ignored + * If an upload fails it can't be retried + +#### --vfs-cache-mode minimal + +This is very similar to "off" except that files opened for read AND +write will be buffered to disks. This means that files opened for +write will be a lot more compatible, but uses the minimal disk space. + +These operations are not possible + + * Files opened for write only can't be seeked + * Existing files opened for write must have O_TRUNC set + * Files opened for write only will ignore O_APPEND, O_TRUNC + * If an upload fails it can't be retried + +#### --vfs-cache-mode writes + +In this mode files opened for read only are still read directly from +the remote, write only and read/write files are buffered to disk +first. + +This mode should support all normal file system operations. + +If an upload fails it will be retried up to --low-level-retries times. + +#### --vfs-cache-mode full + +In this mode all reads and writes are buffered to and from disk. When +a file is opened for read it will be downloaded in its entirety first. + +This may be appropriate for your needs, or you may prefer to look at +the cache backend which does a much more sophisticated job of caching, +including caching directory hierarchies and chunks of files. + +In this mode, unlike the others, when a file is written to the disk, +it will be kept on the disk after it is written to the remote. It +will be purged on a schedule according to `--vfs-cache-max-age`. + +This mode should support all normal file system operations. + +If an upload or download fails it will be retried up to +--low-level-retries times. + + +``` +rclone serve http remote:path [flags] +``` + +### Options + +``` + --addr string IPaddress:Port or :Port to bind server to. (default "localhost:8080") + --cert string SSL PEM key (concatenation of certificate and CA certificate) + --client-ca string Client certificate authority to verify clients with + --dir-cache-time duration Time to cache directory entries for. (default 5m0s) + --gid uint32 Override the gid field set by the filesystem. (default 502) + -h, --help help for http + --htpasswd string htpasswd file - if not provided no authentication is done + --key string SSL PEM Private key + --max-header-bytes int Maximum size of request header (default 4096) + --no-checksum Don't compare checksums on up/download. + --no-modtime Don't read/write the modification time (can speed things up). + --no-seek Don't allow seeking in files. + --pass string Password for authentication. + --poll-interval duration Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable. (default 1m0s) + --read-only Mount read-only. + --realm string realm for authentication (default "rclone") + --server-read-timeout duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --uid uint32 Override the uid field set by the filesystem. (default 502) + --umask int Override the permission bits set by the filesystem. (default 2) + --user string User name for authentication. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + --vfs-read-chunk-size int Read the source objects in chunks. (default 128M) + --vfs-read-chunk-size-limit int If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off) +``` + +## rclone serve restic + +Serve the remote for restic's REST API. + +### Synopsis + +rclone serve restic implements restic's REST backend API +over HTTP. This allows restic to use rclone as a data storage +mechanism for cloud providers that restic does not support directly. + +[Restic](https://restic.net/) is a command line program for doing +backups. + +The server will log errors. Use -v to see access logs. + +--bwlimit will be respected for file transfers. Use --stats to +control the stats printing. + +### Setting up rclone for use by restic ### + +First [set up a remote for your chosen cloud provider](/docs/#configure). + +Once you have set up the remote, check it is working with, for example +"rclone lsd remote:". You may have called the remote something other +than "remote:" - just substitute whatever you called it in the +following instructions. + +Now start the rclone restic server + + rclone serve restic -v remote:backup + +Where you can replace "backup" in the above by whatever path in the +remote you wish to use. + +By default this will serve on "localhost:8080" you can change this +with use of the "--addr" flag. + +You might wish to start this server on boot. + +### Setting up restic to use rclone ### + +Now you can [follow the restic +instructions](http://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) +on setting up restic. + +Note that you will need restic 0.8.2 or later to interoperate with +rclone. + +For the example above you will want to use "http://localhost:8080/" as +the URL for the REST server. + +For example: + + $ export RESTIC_REPOSITORY=rest:http://localhost:8080/ + $ export RESTIC_PASSWORD=yourpassword + $ restic init + created restic backend 8b1a4b56ae at rest:http://localhost:8080/ + + Please note that knowledge of your password is required to access + the repository. Losing your password means that your data is + irrecoverably lost. + $ restic backup /path/to/files/to/backup + scan [/path/to/files/to/backup] + scanned 189 directories, 312 files in 0:00 + [0:00] 100.00% 38.128 MiB / 38.128 MiB 501 / 501 items 0 errors ETA 0:00 + duration: 0:00 + snapshot 45c8fdd8 saved + +#### Multiple repositories #### + +Note that you can use the endpoint to host multiple repositories. Do +this by adding a directory name or path after the URL. Note that +these **must** end with /. Eg + + $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user1repo/ + # backup user1 stuff + $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user2repo/ + # backup user2 stuff + + +### Server options + +Use --addr to specify which IP address and port the server should +listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all +IPs. By default it only listens on localhost. You can use port +:0 to let the OS choose an available port. + +If you set --addr to listen on a public or LAN accessible IP address +then using Authentication is advised - see the next section for info. + +--server-read-timeout and --server-write-timeout can be used to +control the timeouts on the server. Note that this is the total time +for a transfer. + +--max-header-bytes controls the maximum number of bytes the server will +accept in the HTTP header. + +#### Authentication + +By default this will serve files without needing a login. + +You can either use an htpasswd file which can take lots of users, or +set a single username and password with the --user and --pass flags. + +Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is +in standard apache format and supports MD5, SHA1 and BCrypt for basic +authentication. Bcrypt is recommended. + +To create an htpasswd file: + + touch htpasswd + htpasswd -B htpasswd user + htpasswd -B htpasswd anotherUser + +The password file can be updated while rclone is running. + +Use --realm to set the authentication realm. + +#### SSL/TLS + +By default this will serve over http. If you want you can serve over +https. You will need to supply the --cert and --key flags. If you +wish to do client side certificate validation then you will need to +supply --client-ca also. + +--cert should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. --key should be the PEM encoded +private key and --client-ca should be the PEM encoded client +certificate authority certificate. + + +``` +rclone serve restic remote:path [flags] +``` + +### Options + +``` + --addr string IPaddress:Port or :Port to bind server to. (default "localhost:8080") + --append-only disallow deletion of repository data + --cert string SSL PEM key (concatenation of certificate and CA certificate) + --client-ca string Client certificate authority to verify clients with + -h, --help help for restic + --htpasswd string htpasswd file - if not provided no authentication is done + --key string SSL PEM Private key + --max-header-bytes int Maximum size of request header (default 4096) + --pass string Password for authentication. + --realm string realm for authentication (default "rclone") + --server-read-timeout duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --stdio run an HTTP2 server on stdin/stdout + --user string User name for authentication. +``` + +## rclone serve webdav + +Serve remote:path over webdav. + +### Synopsis + + +rclone serve webdav implements a basic webdav server to serve the +remote over HTTP via the webdav protocol. This can be viewed with a +webdav client or you can make a remote of type webdav to read and +write it. + +### Webdav options + +#### --etag-hash + +This controls the ETag header. Without this flag the ETag will be +based on the ModTime and Size of the object. + +If this flag is set to "auto" then rclone will choose the first +supported hash on the backend or you can use a named hash such as +"MD5" or "SHA-1". + +Use "rclone hashsum" to see the full list. + + +### Server options + +Use --addr to specify which IP address and port the server should +listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all +IPs. By default it only listens on localhost. You can use port +:0 to let the OS choose an available port. + +If you set --addr to listen on a public or LAN accessible IP address +then using Authentication is advised - see the next section for info. + +--server-read-timeout and --server-write-timeout can be used to +control the timeouts on the server. Note that this is the total time +for a transfer. + +--max-header-bytes controls the maximum number of bytes the server will +accept in the HTTP header. + +#### Authentication + +By default this will serve files without needing a login. + +You can either use an htpasswd file which can take lots of users, or +set a single username and password with the --user and --pass flags. + +Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is +in standard apache format and supports MD5, SHA1 and BCrypt for basic +authentication. Bcrypt is recommended. + +To create an htpasswd file: + + touch htpasswd + htpasswd -B htpasswd user + htpasswd -B htpasswd anotherUser + +The password file can be updated while rclone is running. + +Use --realm to set the authentication realm. + +#### SSL/TLS + +By default this will serve over http. If you want you can serve over +https. You will need to supply the --cert and --key flags. If you +wish to do client side certificate validation then you will need to +supply --client-ca also. + +--cert should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. --key should be the PEM encoded +private key and --client-ca should be the PEM encoded client +certificate authority certificate. + +### Directory Cache + +Using the `--dir-cache-time` flag, you can set how long a +directory should be considered up to date and not refreshed from the +backend. Changes made locally in the mount may appear immediately or +invalidate the cache. However, changes done on the remote will only +be picked up once the cache expires. + +Alternatively, you can send a `SIGHUP` signal to rclone for +it to flush all directory caches, regardless of how old they are. +Assuming only one rclone instance is running, you can reset the cache +like this: + + kill -SIGHUP $(pidof rclone) + +If you configure rclone with a [remote control](/rc) then you can use +rclone rc to flush the whole directory cache: + + rclone rc vfs/forget + +Or individual files or directories: + + rclone rc vfs/forget file=path/to/file dir=path/to/dir + +### File Buffering + +The `--buffer-size` flag determines the amount of memory, +that will be used to buffer data in advance. + +Each open file descriptor will try to keep the specified amount of +data in memory at all times. The buffered data is bound to one file +descriptor and won't be shared between multiple open file descriptors +of the same file. + +This flag is a upper limit for the used memory per file descriptor. +The buffer will only use memory for data that is downloaded but not +not yet read. If the buffer is empty, only a small amount of memory +will be used. +The maximum memory used by rclone for buffering can be up to +`--buffer-size * open files`. + +### File Caching + +**NB** File caching is **EXPERIMENTAL** - use with care! + +These flags control the VFS file caching options. The VFS layer is +used by rclone mount to make a cloud storage system work more like a +normal file system. + +You'll need to enable VFS caching if you want, for example, to read +and write simultaneously to a file. See below for more details. + +Note that the VFS cache works in addition to the cache backend and you +may find that you need one or the other or both. + + --cache-dir string Directory rclone will use for caching. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + +If run with `-vv` rclone will print the location of the file cache. The +files are stored in the user cache file area which is OS dependent but +can be controlled with `--cache-dir` or setting the appropriate +environment variable. + +The cache has 4 different modes selected by `--vfs-cache-mode`. +The higher the cache mode the more compatible rclone becomes at the +cost of using disk space. + +Note that files are written back to the remote only when they are +closed so if rclone is quit or dies with open files then these won't +get written back to the remote. However they will still be in the on +disk cache. + +#### --vfs-cache-mode off + +In this mode the cache will read directly from the remote and write +directly to the remote without caching anything on disk. + +This will mean some operations are not possible + + * Files can't be opened for both read AND write + * Files opened for write can't be seeked + * Existing files opened for write must have O_TRUNC set + * Files open for read with O_TRUNC will be opened write only + * Files open for write only will behave as if O_TRUNC was supplied + * Open modes O_APPEND, O_TRUNC are ignored + * If an upload fails it can't be retried + +#### --vfs-cache-mode minimal + +This is very similar to "off" except that files opened for read AND +write will be buffered to disks. This means that files opened for +write will be a lot more compatible, but uses the minimal disk space. + +These operations are not possible + + * Files opened for write only can't be seeked + * Existing files opened for write must have O_TRUNC set + * Files opened for write only will ignore O_APPEND, O_TRUNC + * If an upload fails it can't be retried + +#### --vfs-cache-mode writes + +In this mode files opened for read only are still read directly from +the remote, write only and read/write files are buffered to disk +first. + +This mode should support all normal file system operations. + +If an upload fails it will be retried up to --low-level-retries times. + +#### --vfs-cache-mode full + +In this mode all reads and writes are buffered to and from disk. When +a file is opened for read it will be downloaded in its entirety first. + +This may be appropriate for your needs, or you may prefer to look at +the cache backend which does a much more sophisticated job of caching, +including caching directory hierarchies and chunks of files. + +In this mode, unlike the others, when a file is written to the disk, +it will be kept on the disk after it is written to the remote. It +will be purged on a schedule according to `--vfs-cache-max-age`. + +This mode should support all normal file system operations. + +If an upload or download fails it will be retried up to +--low-level-retries times. + + +``` +rclone serve webdav remote:path [flags] +``` + +### Options + +``` + --addr string IPaddress:Port or :Port to bind server to. (default "localhost:8080") + --cert string SSL PEM key (concatenation of certificate and CA certificate) + --client-ca string Client certificate authority to verify clients with + --dir-cache-time duration Time to cache directory entries for. (default 5m0s) + --etag-hash string Which hash to use for the ETag, or auto or blank for off + --gid uint32 Override the gid field set by the filesystem. (default 502) + -h, --help help for webdav + --htpasswd string htpasswd file - if not provided no authentication is done + --key string SSL PEM Private key + --max-header-bytes int Maximum size of request header (default 4096) + --no-checksum Don't compare checksums on up/download. + --no-modtime Don't read/write the modification time (can speed things up). + --no-seek Don't allow seeking in files. + --pass string Password for authentication. + --poll-interval duration Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable. (default 1m0s) + --read-only Mount read-only. + --realm string realm for authentication (default "rclone") + --server-read-timeout duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --uid uint32 Override the uid field set by the filesystem. (default 502) + --umask int Override the permission bits set by the filesystem. (default 2) + --user string User name for authentication. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + --vfs-read-chunk-size int Read the source objects in chunks. (default 128M) + --vfs-read-chunk-size-limit int If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off) +``` + +## rclone touch + +Create new file or change file modification time. + +### Synopsis + +Create new file or change file modification time. + +``` +rclone touch remote:path [flags] +``` + +### Options + +``` + -h, --help help for touch + -C, --no-create Do not create the file if it does not exist. + -t, --timestamp string Change the modification times to the specified time instead of the current time of day. The argument is of the form 'YYMMDD' (ex. 17.10.30) or 'YYYY-MM-DDTHH:MM:SS' (ex. 2006-01-02T15:04:05) +``` + +## rclone tree + +List the contents of the remote in a tree like fashion. + +### Synopsis + + +rclone tree lists the contents of a remote in a similar way to the +unix tree command. + +For example + + $ rclone tree remote:path + / + ├── file1 + ├── file2 + ├── file3 + └── subdir + ├── file4 + └── file5 + + 1 directories, 5 files + +You can use any of the filtering options with the tree command (eg +--include and --exclude). You can also use --fast-list. + +The tree command has many options for controlling the listing which +are compatible with the tree command. Note that not all of them have +short options as they conflict with rclone's short options. + + +``` +rclone tree remote:path [flags] +``` + +### Options + +``` + -a, --all All files are listed (list . files too). + -C, --color Turn colorization on always. + -d, --dirs-only List directories only. + --dirsfirst List directories before files (-U disables). + --full-path Print the full path prefix for each file. + -h, --help help for tree + --human Print the size in a more human readable way. + --level int Descend only level directories deep. + -D, --modtime Print the date of last modification. + -i, --noindent Don't print indentation lines. + --noreport Turn off file/directory count at end of tree listing. + -o, --output string Output to file instead of stdout. + -p, --protections Print the protections for each file. + -Q, --quote Quote filenames with double quotes. + -s, --size Print the size in bytes of each file. + --sort string Select sort: name,version,size,mtime,ctime. + --sort-ctime Sort files by last status change time. + -t, --sort-modtime Sort files by last modification time. + -r, --sort-reverse Reverse the order of the sort. + -U, --unsorted Leave files unsorted. + --version Sort files alphanumerically by version. +``` + + +Copying single files +-------------------- + +rclone normally syncs or copies directories. However, if the source +remote points to a file, rclone will just copy that file. The +destination remote must point to a directory - rclone will give the +error `Failed to create file system for "remote:file": is a file not a +directory` if it isn't. + +For example, suppose you have a remote with a file in called +`test.jpg`, then you could copy just that file like this + + rclone copy remote:test.jpg /tmp/download + +The file `test.jpg` will be placed inside `/tmp/download`. + +This is equivalent to specifying + + rclone copy --files-from /tmp/files remote: /tmp/download + +Where `/tmp/files` contains the single line + + test.jpg + +It is recommended to use `copy` when copying individual files, not `sync`. +They have pretty much the same effect but `copy` will use a lot less +memory. + +Syntax of remote paths +---------------------- + +The syntax of the paths passed to the rclone command are as follows. + +### /path/to/dir + +This refers to the local file system. + +On Windows only `\` may be used instead of `/` in local paths +**only**, non local paths must use `/`. + +These paths needn't start with a leading `/` - if they don't then they +will be relative to the current directory. + +### remote:path/to/dir + +This refers to a directory `path/to/dir` on `remote:` as defined in +the config file (configured with `rclone config`). + +### remote:/path/to/dir + +On most backends this is refers to the same directory as +`remote:path/to/dir` and that format should be preferred. On a very +small number of remotes (FTP, SFTP, Dropbox for business) this will +refer to a different directory. On these, paths without a leading `/` +will refer to your "home" directory and paths with a leading `/` will +refer to the root. + +### :backend:path/to/dir + +This is an advanced form for creating remotes on the fly. `backend` +should be the name or prefix of a backend (the `type` in the config +file) and all the configuration for the backend should be provided on +the command line (or in environment variables). + +Eg + + rclone lsd --http-url https://pub.rclone.org :http: + +Which lists all the directories in `pub.rclone.org`. + +Quoting and the shell +--------------------- + +When you are typing commands to your computer you are using something +called the command line shell. This interprets various characters in +an OS specific way. + +Here are some gotchas which may help users unfamiliar with the shell rules + +### Linux / OSX ### + +If your names have spaces or shell metacharacters (eg `*`, `?`, `$`, +`'`, `"` etc) then you must quote them. Use single quotes `'` by default. + + rclone copy 'Important files?' remote:backup + +If you want to send a `'` you will need to use `"`, eg + + rclone copy "O'Reilly Reviews" remote:backup + +The rules for quoting metacharacters are complicated and if you want +the full details you'll have to consult the manual page for your +shell. + +### Windows ### + +If your names have spaces in you need to put them in `"`, eg + + rclone copy "E:\folder name\folder name\folder name" remote:backup + +If you are using the root directory on its own then don't quote it +(see [#464](https://github.com/ncw/rclone/issues/464) for why), eg + + rclone copy E:\ remote:backup + +Copying files or directories with `:` in the names +-------------------------------------------------- + +rclone uses `:` to mark a remote name. This is, however, a valid +filename component in non-Windows OSes. The remote name parser will +only search for a `:` up to the first `/` so if you need to act on a +file or directory like this then use the full path starting with a +`/`, or use `./` as a current directory prefix. + +So to sync a directory called `sync:me` to a remote called `remote:` use + + rclone sync ./sync:me remote:path + +or + + rclone sync /full/path/to/sync:me remote:path + +Server Side Copy +---------------- + +Most remotes (but not all - see [the +overview](/overview/#optional-features)) support server side copy. + +This means if you want to copy one folder to another then rclone won't +download all the files and re-upload them; it will instruct the server +to copy them in place. + +Eg + + rclone copy s3:oldbucket s3:newbucket + +Will copy the contents of `oldbucket` to `newbucket` without +downloading and re-uploading. + +Remotes which don't support server side copy **will** download and +re-upload in this case. + +Server side copies are used with `sync` and `copy` and will be +identified in the log when using the `-v` flag. The `move` command +may also use them if remote doesn't support server side move directly. +This is done by issuing a server side copy then a delete which is much +quicker than a download and re-upload. + +Server side copies will only be attempted if the remote names are the +same. + +This can be used when scripting to make aged backups efficiently, eg + + rclone sync remote:current-backup remote:previous-backup + rclone sync /path/to/files remote:current-backup + +Options +------- + +Rclone has a number of options to control its behaviour. + +Options which use TIME use the go time parser. A duration string is a +possibly signed sequence of decimal numbers, each with optional +fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid +time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + +Options which use SIZE use kByte by default. However, a suffix of `b` +for bytes, `k` for kBytes, `M` for MBytes, `G` for GBytes, `T` for +TBytes and `P` for PBytes may be used. These are the binary units, eg +1, 2\*\*10, 2\*\*20, 2\*\*30 respectively. + +### --backup-dir=DIR ### + +When using `sync`, `copy` or `move` any files which would have been +overwritten or deleted are moved in their original hierarchy into this +directory. + +If `--suffix` is set, then the moved files will have the suffix added +to them. If there is a file with the same path (after the suffix has +been added) in DIR, then it will be overwritten. + +The remote in use must support server side move or copy and you must +use the same remote as the destination of the sync. The backup +directory must not overlap the destination directory. + +For example + + rclone sync /path/to/local remote:current --backup-dir remote:old + +will sync `/path/to/local` to `remote:current`, but for any files +which would have been updated or deleted will be stored in +`remote:old`. + +If running rclone from a script you might want to use today's date as +the directory name passed to `--backup-dir` to store the old files, or +you might want to pass `--suffix` with today's date. + +### --bind string ### + +Local address to bind to for outgoing connections. This can be an +IPv4 address (1.2.3.4), an IPv6 address (1234::789A) or host name. If +the host name doesn't resolve or resolves to more than one IP address +it will give an error. + +### --bwlimit=BANDWIDTH_SPEC ### + +This option controls the bandwidth limit. Limits can be specified +in two ways: As a single limit, or as a timetable. + +Single limits last for the duration of the session. To use a single limit, +specify the desired bandwidth in kBytes/s, or use a suffix b|k|M|G. The +default is `0` which means to not limit bandwidth. + +For example, to limit bandwidth usage to 10 MBytes/s use `--bwlimit 10M` + +It is also possible to specify a "timetable" of limits, which will cause +certain limits to be applied at certain times. To specify a timetable, format your +entries as "WEEKDAY-HH:MM,BANDWIDTH WEEKDAY-HH:MM,BANDWIDTH..." where: +WEEKDAY is optional element. +It could be writen as whole world or only using 3 first characters. +HH:MM is an hour from 00:00 to 23:59. + +An example of a typical timetable to avoid link saturation during daytime +working hours could be: + +`--bwlimit "08:00,512 12:00,10M 13:00,512 18:00,30M 23:00,off"` + +In this example, the transfer bandwidth will be every day set to 512kBytes/sec at 8am. +At noon, it will raise to 10Mbytes/s, and drop back to 512kBytes/sec at 1pm. +At 6pm, the bandwidth limit will be set to 30MBytes/s, and at 11pm it will be +completely disabled (full speed). Anything between 11pm and 8am will remain +unlimited. + +An example of timetable with WEEKDAY could be: + +`--bwlimit "Mon-00:00,512 Fri-23:59,10M Sat-10:00,1M Sun-20:00,off"` + +It mean that, the transfer bandwidh will be set to 512kBytes/sec on Monday. +It will raise to 10Mbytes/s before the end of Friday. +At 10:00 on Sunday it will be set to 1Mbyte/s. +From 20:00 at Sunday will be unlimited. + +Timeslots without weekday are extended to whole week. +So this one example: + +`--bwlimit "Mon-00:00,512 12:00,1M Sun-20:00,off"` + +Is equal to this: + +`--bwlimit "Mon-00:00,512Mon-12:00,1M Tue-12:00,1M Wed-12:00,1M Thu-12:00,1M Fri-12:00,1M Sat-12:00,1M Sun-12:00,1M Sun-20:00,off"` + +Bandwidth limits only apply to the data transfer. They don't apply to the +bandwidth of the directory listings etc. + +Note that the units are Bytes/s, not Bits/s. Typically connections are +measured in Bits/s - to convert divide by 8. For example, let's say +you have a 10 Mbit/s connection and you wish rclone to use half of it +- 5 Mbit/s. This is 5/8 = 0.625MByte/s so you would use a `--bwlimit +0.625M` parameter for rclone. + +On Unix systems (Linux, MacOS, …) the bandwidth limiter can be toggled by +sending a `SIGUSR2` signal to rclone. This allows to remove the limitations +of a long running rclone transfer and to restore it back to the value specified +with `--bwlimit` quickly when needed. Assuming there is only one rclone instance +running, you can toggle the limiter like this: + + kill -SIGUSR2 $(pidof rclone) + +If you configure rclone with a [remote control](/rc) then you can use +change the bwlimit dynamically: + + rclone rc core/bwlimit rate=1M + +### --buffer-size=SIZE ### + +Use this sized buffer to speed up file transfers. Each `--transfer` +will use this much memory for buffering. + +When using `mount` or `cmount` each open file descriptor will use this much +memory for buffering. +See the [mount](/commands/rclone_mount/#file-buffering) documentation for more details. + +Set to 0 to disable the buffering for the minimum memory usage. + +### --checkers=N ### + +The number of checkers to run in parallel. Checkers do the equality +checking of files during a sync. For some storage systems (eg S3, +Swift, Dropbox) this can take a significant amount of time so they are +run in parallel. + +The default is to run 8 checkers in parallel. + +### -c, --checksum ### + +Normally rclone will look at modification time and size of files to +see if they are equal. If you set this flag then rclone will check +the file hash and size to determine if files are equal. + +This is useful when the remote doesn't support setting modified time +and a more accurate sync is desired than just checking the file size. + +This is very useful when transferring between remotes which store the +same hash type on the object, eg Drive and Swift. For details of which +remotes support which hash type see the table in the [overview +section](https://rclone.org/overview/). + +Eg `rclone --checksum sync s3:/bucket swift:/bucket` would run much +quicker than without the `--checksum` flag. + +When using this flag, rclone won't update mtimes of remote files if +they are incorrect as it would normally. + +### --config=CONFIG_FILE ### + +Specify the location of the rclone config file. + +Normally the config file is in your home directory as a file called +`.config/rclone/rclone.conf` (or `.rclone.conf` if created with an +older version). If `$XDG_CONFIG_HOME` is set it will be at +`$XDG_CONFIG_HOME/rclone/rclone.conf` + +If you run `rclone -h` and look at the help for the `--config` option +you will see where the default location is for you. + +Use this flag to override the config location, eg `rclone +--config=".myconfig" .config`. + +### --contimeout=TIME ### + +Set the connection timeout. This should be in go time format which +looks like `5s` for 5 seconds, `10m` for 10 minutes, or `3h30m`. + +The connection timeout is the amount of time rclone will wait for a +connection to go through to a remote object storage system. It is +`1m` by default. + +### --dedupe-mode MODE ### + +Mode to run dedupe command in. One of `interactive`, `skip`, `first`, `newest`, `oldest`, `rename`. The default is `interactive`. See the dedupe command for more information as to what these options mean. + +### --disable FEATURE,FEATURE,... ### + +This disables a comma separated list of optional features. For example +to disable server side move and server side copy use: + + --disable move,copy + +The features can be put in in any case. + +To see a list of which features can be disabled use: + + --disable help + +See the overview [features](/overview/#features) and +[optional features](/overview/#optional-features) to get an idea of +which feature does what. + +This flag can be useful for debugging and in exceptional circumstances +(eg Google Drive limiting the total volume of Server Side Copies to +100GB/day). + +### -n, --dry-run ### + +Do a trial run with no permanent changes. Use this to see what rclone +would do without actually doing it. Useful when setting up the `sync` +command which deletes files in the destination. + +### --ignore-checksum ### + +Normally rclone will check that the checksums of transferred files +match, and give an error "corrupted on transfer" if they don't. + +You can use this option to skip that check. You should only use it if +you have had the "corrupted on transfer" error message and you are +sure you might want to transfer potentially corrupted data. + +### --ignore-existing ### + +Using this option will make rclone unconditionally skip all files +that exist on the destination, no matter the content of these files. + +While this isn't a generally recommended option, it can be useful +in cases where your files change due to encryption. However, it cannot +correct partial transfers in case a transfer was interrupted. + +### --ignore-size ### + +Normally rclone will look at modification time and size of files to +see if they are equal. If you set this flag then rclone will check +only the modification time. If `--checksum` is set then it only +checks the checksum. + +It will also cause rclone to skip verifying the sizes are the same +after transfer. + +This can be useful for transferring files to and from OneDrive which +occasionally misreports the size of image files (see +[#399](https://github.com/ncw/rclone/issues/399) for more info). + +### -I, --ignore-times ### + +Using this option will cause rclone to unconditionally upload all +files regardless of the state of files on the destination. + +Normally rclone would skip any files that have the same +modification time and are the same size (or have the same checksum if +using `--checksum`). + +### --immutable ### + +Treat source and destination files as immutable and disallow +modification. + +With this option set, files will be created and deleted as requested, +but existing files will never be updated. If an existing file does +not match between the source and destination, rclone will give the error +`Source and destination exist but do not match: immutable file modified`. + +Note that only commands which transfer files (e.g. `sync`, `copy`, +`move`) are affected by this behavior, and only modification is +disallowed. Files may still be deleted explicitly (e.g. `delete`, +`purge`) or implicitly (e.g. `sync`, `move`). Use `copy --immutable` +if it is desired to avoid deletion as well as modification. + +This can be useful as an additional layer of protection for immutable +or append-only data sets (notably backup archives), where modification +implies corruption and should not be propagated. + +## --leave-root ### + +During rmdirs it will not remove root directory, even if it's empty. + +### --log-file=FILE ### + +Log all of rclone's output to FILE. This is not active by default. +This can be useful for tracking down problems with syncs in +combination with the `-v` flag. See the [Logging section](#logging) +for more info. + +Note that if you are using the `logrotate` program to manage rclone's +logs, then you should use the `copytruncate` option as rclone doesn't +have a signal to rotate logs. + +### --log-level LEVEL ### + +This sets the log level for rclone. The default log level is `NOTICE`. + +`DEBUG` is equivalent to `-vv`. It outputs lots of debug info - useful +for bug reports and really finding out what rclone is doing. + +`INFO` is equivalent to `-v`. It outputs information about each transfer +and prints stats once a minute by default. + +`NOTICE` is the default log level if no logging flags are supplied. It +outputs very little when things are working normally. It outputs +warnings and significant events. + +`ERROR` is equivalent to `-q`. It only outputs error messages. + +### --low-level-retries NUMBER ### + +This controls the number of low level retries rclone does. + +A low level retry is used to retry a failing operation - typically one +HTTP request. This might be uploading a chunk of a big file for +example. You will see low level retries in the log with the `-v` +flag. + +This shouldn't need to be changed from the default in normal operations. +However, if you get a lot of low level retries you may wish +to reduce the value so rclone moves on to a high level retry (see the +`--retries` flag) quicker. + +Disable low level retries with `--low-level-retries 1`. + +### --max-backlog=N ### + +This is the maximum allowable backlog of files in a sync/copy/move +queued for being checked or transferred. + +This can be set arbitrarily large. It will only use memory when the +queue is in use. Note that it will use in the order of N kB of memory +when the backlog is in use. + +Setting this large allows rclone to calculate how many files are +pending more accurately and give a more accurate estimated finish +time. + +Setting this small will make rclone more synchronous to the listings +of the remote which may be desirable. + +### --max-delete=N ### + +This tells rclone not to delete more than N files. If that limit is +exceeded then a fatal error will be generated and rclone will stop the +operation in progress. + +### --max-depth=N ### + +This modifies the recursion depth for all the commands except purge. + +So if you do `rclone --max-depth 1 ls remote:path` you will see only +the files in the top level directory. Using `--max-depth 2` means you +will see all the files in first two directory levels and so on. + +For historical reasons the `lsd` command defaults to using a +`--max-depth` of 1 - you can override this with the command line flag. + +You can use this command to disable recursion (with `--max-depth 1`). + +Note that if you use this with `sync` and `--delete-excluded` the +files not recursed through are considered excluded and will be deleted +on the destination. Test first with `--dry-run` if you are not sure +what will happen. + +### --max-transfer=SIZE ### + +Rclone will stop transferring when it has reached the size specified. +Defaults to off. + +When the limit is reached all transfers will stop immediately. + +Rclone will exit with exit code 8 if the transfer limit is reached. + +### --modify-window=TIME ### + +When checking whether a file has been modified, this is the maximum +allowed time difference that a file can have and still be considered +equivalent. + +The default is `1ns` unless this is overridden by a remote. For +example OS X only stores modification times to the nearest second so +if you are reading and writing to an OS X filing system this will be +`1s` by default. + +This command line flag allows you to override that computed default. + +### --no-gzip-encoding ### + +Don't set `Accept-Encoding: gzip`. This means that rclone won't ask +the server for compressed files automatically. Useful if you've set +the server to return files with `Content-Encoding: gzip` but you +uploaded compressed files. + +There is no need to set this in normal operation, and doing so will +decrease the network transfer efficiency of rclone. + +### --no-update-modtime ### + +When using this flag, rclone won't update modification times of remote +files if they are incorrect as it would normally. + +This can be used if the remote is being synced with another tool also +(eg the Google Drive client). + +### --P, --progress ### + +This flag makes rclone update the stats in a static block in the +terminal providing a realtime overview of the transfer. + +Any log messages will scroll above the static block. Log messages +will push the static block down to the bottom of the terminal where it +will stay. + +Normally this is updated every 500mS but this period can be overridden +with the `--stats` flag. + +This can be used with the `--stats-one-line` flag for a simpler +display. + +### -q, --quiet ### + +Normally rclone outputs stats and a completion message. If you set +this flag it will make as little output as possible. + +### --retries int ### + +Retry the entire sync if it fails this many times it fails (default 3). + +Some remotes can be unreliable and a few retries help pick up the +files which didn't get transferred because of errors. + +Disable retries with `--retries 1`. + +### --retries-sleep=TIME ### + +This sets the interval between each retry specified by `--retries` + +The default is 0. Use 0 to disable. + +### --size-only ### + +Normally rclone will look at modification time and size of files to +see if they are equal. If you set this flag then rclone will check +only the size. + +This can be useful transferring files from Dropbox which have been +modified by the desktop sync client which doesn't set checksums of +modification times in the same way as rclone. + +### --stats=TIME ### + +Commands which transfer data (`sync`, `copy`, `copyto`, `move`, +`moveto`) will print data transfer stats at regular intervals to show +their progress. + +This sets the interval. + +The default is `1m`. Use 0 to disable. + +If you set the stats interval then all commands can show stats. This +can be useful when running other commands, `check` or `mount` for +example. + +Stats are logged at `INFO` level by default which means they won't +show at default log level `NOTICE`. Use `--stats-log-level NOTICE` or +`-v` to make them show. See the [Logging section](#logging) for more +info on log levels. + +Note that on macOS you can send a SIGINFO (which is normally ctrl-T in +the terminal) to make the stats print immediately. + +### --stats-file-name-length integer ### +By default, the `--stats` output will truncate file names and paths longer +than 40 characters. This is equivalent to providing +`--stats-file-name-length 40`. Use `--stats-file-name-length 0` to disable +any truncation of file names printed by stats. + +### --stats-log-level string ### + +Log level to show `--stats` output at. This can be `DEBUG`, `INFO`, +`NOTICE`, or `ERROR`. The default is `INFO`. This means at the +default level of logging which is `NOTICE` the stats won't show - if +you want them to then use `--stats-log-level NOTICE`. See the [Logging +section](#logging) for more info on log levels. + +### --stats-one-line ### + +When this is specified, rclone condenses the stats into a single line +showing the most important stats only. + +### --stats-unit=bits|bytes ### + +By default, data transfer rates will be printed in bytes/second. + +This option allows the data rate to be printed in bits/second. + +Data transfer volume will still be reported in bytes. + +The rate is reported as a binary unit, not SI unit. So 1 Mbit/s +equals 1,048,576 bits/s and not 1,000,000 bits/s. + +The default is `bytes`. + +### --suffix=SUFFIX ### + +This is for use with `--backup-dir` only. If this isn't set then +`--backup-dir` will move files with their original name. If it is set +then the files will have SUFFIX added on to them. + +See `--backup-dir` for more info. + +### --syslog ### + +On capable OSes (not Windows or Plan9) send all log output to syslog. + +This can be useful for running rclone in a script or `rclone mount`. + +### --syslog-facility string ### + +If using `--syslog` this sets the syslog facility (eg `KERN`, `USER`). +See `man syslog` for a list of possible facilities. The default +facility is `DAEMON`. + +### --tpslimit float ### + +Limit HTTP transactions per second to this. Default is 0 which is used +to mean unlimited transactions per second. + +For example to limit rclone to 10 HTTP transactions per second use +`--tpslimit 10`, or to 1 transaction every 2 seconds use `--tpslimit +0.5`. + +Use this when the number of transactions per second from rclone is +causing a problem with the cloud storage provider (eg getting you +banned or rate limited). + +This can be very useful for `rclone mount` to control the behaviour of +applications using it. + +See also `--tpslimit-burst`. + +### --tpslimit-burst int ### + +Max burst of transactions for `--tpslimit`. (default 1) + +Normally `--tpslimit` will do exactly the number of transaction per +second specified. However if you supply `--tps-burst` then rclone can +save up some transactions from when it was idle giving a burst of up +to the parameter supplied. + +For example if you provide `--tpslimit-burst 10` then if rclone has +been idle for more than 10*`--tpslimit` then it can do 10 transactions +very quickly before they are limited again. + +This may be used to increase performance of `--tpslimit` without +changing the long term average number of transactions per second. + +### --track-renames ### + +By default, rclone doesn't keep track of renamed files, so if you +rename a file locally then sync it to a remote, rclone will delete the +old file on the remote and upload a new copy. + +If you use this flag, and the remote supports server side copy or +server side move, and the source and destination have a compatible +hash, then this will track renames during `sync` +operations and perform renaming server-side. + +Files will be matched by size and hash - if both match then a rename +will be considered. + +If the destination does not support server-side copy or move, rclone +will fall back to the default behaviour and log an error level message +to the console. + +Note that `--track-renames` uses extra memory to keep track of all +the rename candidates. + +Note also that `--track-renames` is incompatible with +`--delete-before` and will select `--delete-after` instead of +`--delete-during`. + +### --delete-(before,during,after) ### + +This option allows you to specify when files on your destination are +deleted when you sync folders. + +Specifying the value `--delete-before` will delete all files present +on the destination, but not on the source *before* starting the +transfer of any new or updated files. This uses two passes through the +file systems, one for the deletions and one for the copies. + +Specifying `--delete-during` will delete files while checking and +uploading files. This is the fastest option and uses the least memory. + +Specifying `--delete-after` (the default value) will delay deletion of +files until all new/updated files have been successfully transferred. +The files to be deleted are collected in the copy pass then deleted +after the copy pass has completed successfully. The files to be +deleted are held in memory so this mode may use more memory. This is +the safest mode as it will only delete files if there have been no +errors subsequent to that. If there have been errors before the +deletions start then you will get the message `not deleting files as +there were IO errors`. + +### --fast-list ### + +When doing anything which involves a directory listing (eg `sync`, +`copy`, `ls` - in fact nearly every command), rclone normally lists a +directory and processes it before using more directory lists to +process any subdirectories. This can be parallelised and works very +quickly using the least amount of memory. + +However, some remotes have a way of listing all files beneath a +directory in one (or a small number) of transactions. These tend to +be the bucket based remotes (eg S3, B2, GCS, Swift, Hubic). + +If you use the `--fast-list` flag then rclone will use this method for +listing directories. This will have the following consequences for +the listing: + + * It **will** use fewer transactions (important if you pay for them) + * It **will** use more memory. Rclone has to load the whole listing into memory. + * It *may* be faster because it uses fewer transactions + * It *may* be slower because it can't be parallelized + +rclone should always give identical results with and without +`--fast-list`. + +If you pay for transactions and can fit your entire sync listing into +memory then `--fast-list` is recommended. If you have a very big sync +to do then don't use `--fast-list` otherwise you will run out of +memory. + +If you use `--fast-list` on a remote which doesn't support it, then +rclone will just ignore it. + +### --timeout=TIME ### + +This sets the IO idle timeout. If a transfer has started but then +becomes idle for this long it is considered broken and disconnected. + +The default is `5m`. Set to 0 to disable. + +### --transfers=N ### + +The number of file transfers to run in parallel. It can sometimes be +useful to set this to a smaller number if the remote is giving a lot +of timeouts or bigger if you have lots of bandwidth and a fast remote. + +The default is to run 4 file transfers in parallel. + +### -u, --update ### + +This forces rclone to skip any files which exist on the destination +and have a modified time that is newer than the source file. + +If an existing destination file has a modification time equal (within +the computed modify window precision) to the source file's, it will be +updated if the sizes are different. + +On remotes which don't support mod time directly the time checked will +be the uploaded time. This means that if uploading to one of these +remotes, rclone will skip any files which exist on the destination and +have an uploaded time that is newer than the modification time of the +source file. + +This can be useful when transferring to a remote which doesn't support +mod times directly as it is more accurate than a `--size-only` check +and faster than using `--checksum`. + +### --use-server-modtime ### + +Some object-store backends (e.g, Swift, S3) do not preserve file modification +times (modtime). On these backends, rclone stores the original modtime as +additional metadata on the object. By default it will make an API call to +retrieve the metadata when the modtime is needed by an operation. + +Use this flag to disable the extra API call and rely instead on the server's +modified time. In cases such as a local to remote sync, knowing the local file +is newer than the time it was last uploaded to the remote is sufficient. In +those cases, this flag can speed up the process and reduce the number of API +calls necessary. + +### -v, -vv, --verbose ### + +With `-v` rclone will tell you about each file that is transferred and +a small number of significant events. + +With `-vv` rclone will become very verbose telling you about every +file it considers and transfers. Please send bug reports with a log +with this setting. + +### -V, --version ### + +Prints the version number + +Configuration Encryption +------------------------ +Your configuration file contains information for logging in to +your cloud services. This means that you should keep your +`.rclone.conf` file in a secure location. + +If you are in an environment where that isn't possible, you can +add a password to your configuration. This means that you will +have to enter the password every time you start rclone. + +To add a password to your rclone configuration, execute `rclone config`. + +``` +>rclone config +Current remotes: + +e) Edit existing remote +n) New remote +d) Delete remote +s) Set configuration password +q) Quit config +e/n/d/s/q> +``` + +Go into `s`, Set configuration password: +``` +e/n/d/s/q> s +Your configuration is not encrypted. +If you add a password, you will protect your login information to cloud services. +a) Add Password +q) Quit to main menu +a/q> a +Enter NEW configuration password: +password: +Confirm NEW password: +password: +Password set +Your configuration is encrypted. +c) Change Password +u) Unencrypt configuration +q) Quit to main menu +c/u/q> +``` + +Your configuration is now encrypted, and every time you start rclone +you will now be asked for the password. In the same menu, you can +change the password or completely remove encryption from your +configuration. + +There is no way to recover the configuration if you lose your password. + +rclone uses [nacl secretbox](https://godoc.org/golang.org/x/crypto/nacl/secretbox) +which in turn uses XSalsa20 and Poly1305 to encrypt and authenticate +your configuration with secret-key cryptography. +The password is SHA-256 hashed, which produces the key for secretbox. +The hashed password is not stored. + +While this provides very good security, we do not recommend storing +your encrypted rclone configuration in public if it contains sensitive +information, maybe except if you use a very strong password. + +If it is safe in your environment, you can set the `RCLONE_CONFIG_PASS` +environment variable to contain your password, in which case it will be +used for decrypting the configuration. + +You can set this for a session from a script. For unix like systems +save this to a file called `set-rclone-password`: + +``` +#!/bin/echo Source this file don't run it + +read -s RCLONE_CONFIG_PASS +export RCLONE_CONFIG_PASS +``` + +Then source the file when you want to use it. From the shell you +would do `source set-rclone-password`. It will then ask you for the +password and set it in the environment variable. + +If you are running rclone inside a script, you might want to disable +password prompts. To do that, pass the parameter +`--ask-password=false` to rclone. This will make rclone fail instead +of asking for a password if `RCLONE_CONFIG_PASS` doesn't contain +a valid password. + + +Developer options +----------------- + +These options are useful when developing or debugging rclone. There +are also some more remote specific options which aren't documented +here which are used for testing. These start with remote name eg +`--drive-test-option` - see the docs for the remote in question. + +### --cpuprofile=FILE ### + +Write CPU profile to file. This can be analysed with `go tool pprof`. + +#### --dump flag,flag,flag #### + +The `--dump` flag takes a comma separated list of flags to dump info +about. These are: + +#### --dump headers #### + +Dump HTTP headers with `Authorization:` lines removed. May still +contain sensitive info. Can be very verbose. Useful for debugging +only. + +Use `--dump auth` if you do want the `Authorization:` headers. + +#### --dump bodies #### + +Dump HTTP headers and bodies - may contain sensitive info. Can be +very verbose. Useful for debugging only. + +Note that the bodies are buffered in memory so don't use this for +enormous files. + +#### --dump requests #### + +Like `--dump bodies` but dumps the request bodies and the response +headers. Useful for debugging download problems. + +#### --dump responses #### + +Like `--dump bodies` but dumps the response bodies and the request +headers. Useful for debugging upload problems. + +#### --dump auth #### + +Dump HTTP headers - will contain sensitive info such as +`Authorization:` headers - use `--dump headers` to dump without +`Authorization:` headers. Can be very verbose. Useful for debugging +only. + +#### --dump filters #### + +Dump the filters to the output. Useful to see exactly what include +and exclude options are filtering on. + +#### --dump goroutines #### + +This dumps a list of the running go-routines at the end of the command +to standard output. + +#### --dump openfiles #### + +This dumps a list of the open files at the end of the command. It +uses the `lsof` command to do that so you'll need that installed to +use it. + +### --memprofile=FILE ### + +Write memory profile to file. This can be analysed with `go tool pprof`. + +### --no-check-certificate=true/false ### + +`--no-check-certificate` controls whether a client verifies the +server's certificate chain and host name. +If `--no-check-certificate` is true, TLS accepts any certificate +presented by the server and any host name in that certificate. +In this mode, TLS is susceptible to man-in-the-middle attacks. + +This option defaults to `false`. + +**This should be used only for testing.** + +Filtering +--------- + +For the filtering options + + * `--delete-excluded` + * `--filter` + * `--filter-from` + * `--exclude` + * `--exclude-from` + * `--include` + * `--include-from` + * `--files-from` + * `--min-size` + * `--max-size` + * `--min-age` + * `--max-age` + * `--dump filters` + +See the [filtering section](https://rclone.org/filtering/). + +Remote control +-------------- + +For the remote control options and for instructions on how to remote control rclone + + * `--rc` + * and anything starting with `--rc-` + +See [the remote control section](https://rclone.org/rc/). + +Logging +------- + +rclone has 4 levels of logging, `ERROR`, `NOTICE`, `INFO` and `DEBUG`. + +By default, rclone logs to standard error. This means you can redirect +standard error and still see the normal output of rclone commands (eg +`rclone ls`). + +By default, rclone will produce `Error` and `Notice` level messages. + +If you use the `-q` flag, rclone will only produce `Error` messages. + +If you use the `-v` flag, rclone will produce `Error`, `Notice` and +`Info` messages. + +If you use the `-vv` flag, rclone will produce `Error`, `Notice`, +`Info` and `Debug` messages. + +You can also control the log levels with the `--log-level` flag. + +If you use the `--log-file=FILE` option, rclone will redirect `Error`, +`Info` and `Debug` messages along with standard error to FILE. + +If you use the `--syslog` flag then rclone will log to syslog and the +`--syslog-facility` control which facility it uses. + +Rclone prefixes all log messages with their level in capitals, eg INFO +which makes it easy to grep the log file for different kinds of +information. + +Exit Code +--------- + +If any errors occur during the command execution, rclone will exit with a +non-zero exit code. This allows scripts to detect when rclone +operations have failed. + +During the startup phase, rclone will exit immediately if an error is +detected in the configuration. There will always be a log message +immediately before exiting. + +When rclone is running it will accumulate errors as it goes along, and +only exit with a non-zero exit code if (after retries) there were +still failed transfers. For every error counted there will be a high +priority log message (visible with `-q`) showing the message and +which file caused the problem. A high priority message is also shown +when starting a retry so the user can see that any previous error +messages may not be valid after the retry. If rclone has done a retry +it will log a high priority message if the retry was successful. + +### List of exit codes ### + * `0` - success + * `1` - Syntax or usage error + * `2` - Error not otherwise categorised + * `3` - Directory not found + * `4` - File not found + * `5` - Temporary error (one that more retries might fix) (Retry errors) + * `6` - Less serious errors (like 461 errors from dropbox) (NoRetry errors) + * `7` - Fatal error (one that more retries won't fix, like account suspended) (Fatal errors) + * `8` - Transfer exceeded - limit set by --max-transfer reached + +Environment Variables +--------------------- + +Rclone can be configured entirely using environment variables. These +can be used to set defaults for options or config file entries. + +### Options ### + +Every option in rclone can have its default set by environment +variable. + +To find the name of the environment variable, first, take the long +option name, strip the leading `--`, change `-` to `_`, make +upper case and prepend `RCLONE_`. + +For example, to always set `--stats 5s`, set the environment variable +`RCLONE_STATS=5s`. If you set stats on the command line this will +override the environment variable setting. + +Or to always use the trash in drive `--drive-use-trash`, set +`RCLONE_DRIVE_USE_TRASH=true`. + +The same parser is used for the options and the environment variables +so they take exactly the same form. + +### Config file ### + +You can set defaults for values in the config file on an individual +remote basis. If you want to use this feature, you will need to +discover the name of the config items that you want. The easiest way +is to run through `rclone config` by hand, then look in the config +file to see what the values are (the config file can be found by +looking at the help for `--config` in `rclone help`). + +To find the name of the environment variable, you need to set, take +`RCLONE_CONFIG_` + name of remote + `_` + name of config file option +and make it all uppercase. + +For example, to configure an S3 remote named `mys3:` without a config +file (using unix ways of setting environment variables): + +``` +$ export RCLONE_CONFIG_MYS3_TYPE=s3 +$ export RCLONE_CONFIG_MYS3_ACCESS_KEY_ID=XXX +$ export RCLONE_CONFIG_MYS3_SECRET_ACCESS_KEY=XXX +$ rclone lsd MYS3: + -1 2016-09-21 12:54:21 -1 my-bucket +$ rclone listremotes | grep mys3 +mys3: +``` + +Note that if you want to create a remote using environment variables +you must create the `..._TYPE` variable as above. + +### Other environment variables ### + + * RCLONE_CONFIG_PASS` set to contain your config file password (see [Configuration Encryption](#configuration-encryption) section) + * HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions thereof). + * HTTPS_PROXY takes precedence over HTTP_PROXY for https requests. + * The environment values may be either a complete URL or a "host[:port]" for, in which case the "http" scheme is assumed. + +# Configuring rclone on a remote / headless machine # + +Some of the configurations (those involving oauth2) require an +Internet connected web browser. + +If you are trying to set rclone up on a remote or headless box with no +browser available on it (eg a NAS or a server in a datacenter) then +you will need to use an alternative means of configuration. There are +two ways of doing it, described below. + +## Configuring using rclone authorize ## + +On the headless box + +``` +... +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> n +For this to work, you will need rclone available on a machine that has a web browser available. +Execute the following on your machine: + rclone authorize "amazon cloud drive" +Then paste the result below: +result> +``` + +Then on your main desktop machine + +``` +rclone authorize "amazon cloud drive" +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +Paste the following into your remote machine ---> +SECRET_TOKEN +<---End paste +``` + +Then back to the headless box, paste in the code + +``` +result> SECRET_TOKEN +-------------------- +[acd12] +client_id = +client_secret = +token = SECRET_TOKEN +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> +``` + +## Configuring by copying the config file ## + +Rclone stores all of its config in a single configuration file. This +can easily be copied to configure a remote rclone. + +So first configure rclone on your desktop machine + + rclone config + +to set up the config file. + +Find the config file by running `rclone -h` and looking for the help for the `--config` option + +``` +$ rclone -h +[snip] + --config="/home/user/.rclone.conf": Config file. +[snip] +``` + +Now transfer it to the remote box (scp, cut paste, ftp, sftp etc) and +place it in the correct place (use `rclone -h` on the remote box to +find out where). + +# Filtering, includes and excludes # + +Rclone has a sophisticated set of include and exclude rules. Some of +these are based on patterns and some on other things like file size. + +The filters are applied for the `copy`, `sync`, `move`, `ls`, `lsl`, +`md5sum`, `sha1sum`, `size`, `delete` and `check` operations. +Note that `purge` does not obey the filters. + +Each path as it passes through rclone is matched against the include +and exclude rules like `--include`, `--exclude`, `--include-from`, +`--exclude-from`, `--filter`, or `--filter-from`. The simplest way to +try them out is using the `ls` command, or `--dry-run` together with +`-v`. + +## Patterns ## + +The patterns used to match files for inclusion or exclusion are based +on "file globs" as used by the unix shell. + +If the pattern starts with a `/` then it only matches at the top level +of the directory tree, **relative to the root of the remote** (not +necessarily the root of the local drive). If it doesn't start with `/` +then it is matched starting at the **end of the path**, but it will +only match a complete path element: + + file.jpg - matches "file.jpg" + - matches "directory/file.jpg" + - doesn't match "afile.jpg" + - doesn't match "directory/afile.jpg" + /file.jpg - matches "file.jpg" in the root directory of the remote + - doesn't match "afile.jpg" + - doesn't match "directory/file.jpg" + +**Important** Note that you must use `/` in patterns and not `\` even +if running on Windows. + +A `*` matches anything but not a `/`. + + *.jpg - matches "file.jpg" + - matches "directory/file.jpg" + - doesn't match "file.jpg/something" + +Use `**` to match anything, including slashes (`/`). + + dir/** - matches "dir/file.jpg" + - matches "dir/dir1/dir2/file.jpg" + - doesn't match "directory/file.jpg" + - doesn't match "adir/file.jpg" + +A `?` matches any character except a slash `/`. + + l?ss - matches "less" + - matches "lass" + - doesn't match "floss" + +A `[` and `]` together make a a character class, such as `[a-z]` or +`[aeiou]` or `[[:alpha:]]`. See the [go regexp +docs](https://golang.org/pkg/regexp/syntax/) for more info on these. + + h[ae]llo - matches "hello" + - matches "hallo" + - doesn't match "hullo" + +A `{` and `}` define a choice between elements. It should contain a +comma separated list of patterns, any of which might match. These +patterns can contain wildcards. + + {one,two}_potato - matches "one_potato" + - matches "two_potato" + - doesn't match "three_potato" + - doesn't match "_potato" + +Special characters can be escaped with a `\` before them. + + \*.jpg - matches "*.jpg" + \\.jpg - matches "\.jpg" + \[one\].jpg - matches "[one].jpg" + +Note also that rclone filter globs can only be used in one of the +filter command line flags, not in the specification of the remote, so +`rclone copy "remote:dir*.jpg" /path/to/dir` won't work - what is +required is `rclone --include "*.jpg" copy remote:dir /path/to/dir` + +### Directories ### + +Rclone keeps track of directories that could match any file patterns. + +Eg if you add the include rule + + /a/*.jpg + +Rclone will synthesize the directory include rule + + /a/ + +If you put any rules which end in `/` then it will only match +directories. + +Directory matches are **only** used to optimise directory access +patterns - you must still match the files that you want to match. +Directory matches won't optimise anything on bucket based remotes (eg +s3, swift, google compute storage, b2) which don't have a concept of +directory. + +### Differences between rsync and rclone patterns ### + +Rclone implements bash style `{a,b,c}` glob matching which rsync doesn't. + +Rclone always does a wildcard match so `\` must always escape a `\`. + +## How the rules are used ## + +Rclone maintains a combined list of include rules and exclude rules. + +Each file is matched in order, starting from the top, against the rule +in the list until it finds a match. The file is then included or +excluded according to the rule type. + +If the matcher fails to find a match after testing against all the +entries in the list then the path is included. + +For example given the following rules, `+` being include, `-` being +exclude, + + - secret*.jpg + + *.jpg + + *.png + + file2.avi + - * + +This would include + + * `file1.jpg` + * `file3.png` + * `file2.avi` + +This would exclude + + * `secret17.jpg` + * non `*.jpg` and `*.png` + +A similar process is done on directory entries before recursing into +them. This only works on remotes which have a concept of directory +(Eg local, google drive, onedrive, amazon drive) and not on bucket +based remotes (eg s3, swift, google compute storage, b2). + +## Adding filtering rules ## + +Filtering rules are added with the following command line flags. + +### Repeating options ## + +You can repeat the following options to add more than one rule of that +type. + + * `--include` + * `--include-from` + * `--exclude` + * `--exclude-from` + * `--filter` + * `--filter-from` + +**Important** You should not use `--include*` together with `--exclude*`. +It may produce different results than you expected. In that case try to use: `--filter*`. + +Note that all the options of the same type are processed together in +the order above, regardless of what order they were placed on the +command line. + +So all `--include` options are processed first in the order they +appeared on the command line, then all `--include-from` options etc. + +To mix up the order includes and excludes, the `--filter` flag can be +used. + +### `--exclude` - Exclude files matching pattern ### + +Add a single exclude rule with `--exclude`. + +This flag can be repeated. See above for the order the flags are +processed in. + +Eg `--exclude *.bak` to exclude all bak files from the sync. + +### `--exclude-from` - Read exclude patterns from file ### + +Add exclude rules from a file. + +This flag can be repeated. See above for the order the flags are +processed in. + +Prepare a file like this `exclude-file.txt` + + # a sample exclude rule file + *.bak + file2.jpg + +Then use as `--exclude-from exclude-file.txt`. This will sync all +files except those ending in `bak` and `file2.jpg`. + +This is useful if you have a lot of rules. + +### `--include` - Include files matching pattern ### + +Add a single include rule with `--include`. + +This flag can be repeated. See above for the order the flags are +processed in. + +Eg `--include *.{png,jpg}` to include all `png` and `jpg` files in the +backup and no others. + +This adds an implicit `--exclude *` at the very end of the filter +list. This means you can mix `--include` and `--include-from` with the +other filters (eg `--exclude`) but you must include all the files you +want in the include statement. If this doesn't provide enough +flexibility then you must use `--filter-from`. + +### `--include-from` - Read include patterns from file ### + +Add include rules from a file. + +This flag can be repeated. See above for the order the flags are +processed in. + +Prepare a file like this `include-file.txt` + + # a sample include rule file + *.jpg + *.png + file2.avi + +Then use as `--include-from include-file.txt`. This will sync all +`jpg`, `png` files and `file2.avi`. + +This is useful if you have a lot of rules. + +This adds an implicit `--exclude *` at the very end of the filter +list. This means you can mix `--include` and `--include-from` with the +other filters (eg `--exclude`) but you must include all the files you +want in the include statement. If this doesn't provide enough +flexibility then you must use `--filter-from`. + +### `--filter` - Add a file-filtering rule ### + +This can be used to add a single include or exclude rule. Include +rules start with `+ ` and exclude rules start with `- `. A special +rule called `!` can be used to clear the existing rules. + +This flag can be repeated. See above for the order the flags are +processed in. + +Eg `--filter "- *.bak"` to exclude all bak files from the sync. + +### `--filter-from` - Read filtering patterns from a file ### + +Add include/exclude rules from a file. + +This flag can be repeated. See above for the order the flags are +processed in. + +Prepare a file like this `filter-file.txt` + + # a sample filter rule file + - secret*.jpg + + *.jpg + + *.png + + file2.avi + - /dir/Trash/** + + /dir/** + # exclude everything else + - * + +Then use as `--filter-from filter-file.txt`. The rules are processed +in the order that they are defined. + +This example will include all `jpg` and `png` files, exclude any files +matching `secret*.jpg` and include `file2.avi`. It will also include +everything in the directory `dir` at the root of the sync, except +`dir/Trash` which it will exclude. Everything else will be excluded +from the sync. + +### `--files-from` - Read list of source-file names ### + +This reads a list of file names from the file passed in and **only** +these files are transferred. The **filtering rules are ignored** +completely if you use this option. + +This option can be repeated to read from more than one file. These +are read in the order that they are placed on the command line. + +Paths within the `--files-from` file will be interpreted as starting +with the root specified in the command. Leading `/` characters are +ignored. + +For example, suppose you had `files-from.txt` with this content: + + # comment + file1.jpg + subdir/file2.jpg + +You could then use it like this: + + rclone copy --files-from files-from.txt /home/me/pics remote:pics + +This will transfer these files only (if they exist) + + /home/me/pics/file1.jpg → remote:pics/file1.jpg + /home/me/pics/subdir/file2.jpg → remote:pics/subdirfile1.jpg + +To take a more complicated example, let's say you had a few files you +want to back up regularly with these absolute paths: + + /home/user1/important + /home/user1/dir/file + /home/user2/stuff + +To copy these you'd find a common subdirectory - in this case `/home` +and put the remaining files in `files-from.txt` with or without +leading `/`, eg + + user1/important + user1/dir/file + user2/stuff + +You could then copy these to a remote like this + + rclone copy --files-from files-from.txt /home remote:backup + +The 3 files will arrive in `remote:backup` with the paths as in the +`files-from.txt` like this: + + /home/user1/important → remote:backup/user1/important + /home/user1/dir/file → remote:backup/user1/dir/file + /home/user2/stuff → remote:backup/stuff + +You could of course choose `/` as the root too in which case your +`files-from.txt` might look like this. + + /home/user1/important + /home/user1/dir/file + /home/user2/stuff + +And you would transfer it like this + + rclone copy --files-from files-from.txt / remote:backup + +In this case there will be an extra `home` directory on the remote: + + /home/user1/important → remote:home/backup/user1/important + /home/user1/dir/file → remote:home/backup/user1/dir/file + /home/user2/stuff → remote:home/backup/stuff + +### `--min-size` - Don't transfer any file smaller than this ### + +This option controls the minimum size file which will be transferred. +This defaults to `kBytes` but a suffix of `k`, `M`, or `G` can be +used. + +For example `--min-size 50k` means no files smaller than 50kByte will be +transferred. + +### `--max-size` - Don't transfer any file larger than this ### + +This option controls the maximum size file which will be transferred. +This defaults to `kBytes` but a suffix of `k`, `M`, or `G` can be +used. + +For example `--max-size 1G` means no files larger than 1GByte will be +transferred. + +### `--max-age` - Don't transfer any file older than this ### + +This option controls the maximum age of files to transfer. Give in +seconds or with a suffix of: + + * `ms` - Milliseconds + * `s` - Seconds + * `m` - Minutes + * `h` - Hours + * `d` - Days + * `w` - Weeks + * `M` - Months + * `y` - Years + +For example `--max-age 2d` means no files older than 2 days will be +transferred. + +### `--min-age` - Don't transfer any file younger than this ### + +This option controls the minimum age of files to transfer. Give in +seconds or with a suffix (see `--max-age` for list of suffixes) + +For example `--min-age 2d` means no files younger than 2 days will be +transferred. + +### `--delete-excluded` - Delete files on dest excluded from sync ### + +**Important** this flag is dangerous - use with `--dry-run` and `-v` first. + +When doing `rclone sync` this will delete any files which are excluded +from the sync on the destination. + +If for example you did a sync from `A` to `B` without the `--min-size 50k` flag + + rclone sync A: B: + +Then you repeated it like this with the `--delete-excluded` + + rclone --min-size 50k --delete-excluded sync A: B: + +This would delete all files on `B` which are less than 50 kBytes as +these are now excluded from the sync. + +Always test first with `--dry-run` and `-v` before using this flag. + +### `--dump filters` - dump the filters to the output ### + +This dumps the defined filters to the output as regular expressions. + +Useful for debugging. + +## Quoting shell metacharacters ## + +The examples above may not work verbatim in your shell as they have +shell metacharacters in them (eg `*`), and may require quoting. + +Eg linux, OSX + + * `--include \*.jpg` + * `--include '*.jpg'` + * `--include='*.jpg'` + +In Windows the expansion is done by the command not the shell so this +should work fine + + * `--include *.jpg` + +## Exclude directory based on a file ## + +It is possible to exclude a directory based on a file, which is +present in this directory. Filename should be specified using the +`--exclude-if-present` flag. This flag has a priority over the other +filtering flags. + +Imagine, you have the following directory structure: + + dir1/file1 + dir1/dir2/file2 + dir1/dir2/dir3/file3 + dir1/dir2/dir3/.ignore + +You can exclude `dir3` from sync by running the following command: + + rclone sync --exclude-if-present .ignore dir1 remote:backup + +Currently only one filename is supported, i.e. `--exclude-if-present` +should not be used multiple times. + +# Remote controlling rclone # + +If rclone is run with the `--rc` flag then it starts an http server +which can be used to remote control rclone. + +**NB** this is experimental and everything here is subject to change! + +## Supported parameters + +#### --rc #### +Flag to start the http server listen on remote requests + +#### --rc-addr=IP #### +IPaddress:Port or :Port to bind server to. (default "localhost:5572") + +#### --rc-cert=KEY #### +SSL PEM key (concatenation of certificate and CA certificate) + +#### --rc-client-ca=PATH #### +Client certificate authority to verify clients with + +#### --rc-htpasswd=PATH #### +htpasswd file - if not provided no authentication is done + +#### --rc-key=PATH #### +SSL PEM Private key + +#### --rc-max-header-bytes=VALUE #### +Maximum size of request header (default 4096) + +#### --rc-user=VALUE #### +User name for authentication. + +#### --rc-pass=VALUE #### +Password for authentication. + +#### --rc-realm=VALUE #### +Realm for authentication (default "rclone") + +#### --rc-server-read-timeout=DURATION #### +Timeout for server reading data (default 1h0m0s) + +#### --rc-server-write-timeout=DURATION #### +Timeout for server writing data (default 1h0m0s) + +## Accessing the remote control via the rclone rc command + +Rclone itself implements the remote control protocol in its `rclone +rc` command. + +You can use it like this + +``` +$ rclone rc rc/noop param1=one param2=two +{ + "param1": "one", + "param2": "two" +} +``` + +Run `rclone rc` on its own to see the help for the installed remote +control commands. + +## Supported commands + +### cache/expire: Purge a remote from cache + +Purge a remote from the cache backend. Supports either a directory or a file. +Params: + - remote = path to remote (required) + - withData = true/false to delete cached data (chunks) as well (optional) + +Eg + + rclone rc cache/expire remote=path/to/sub/folder/ + rclone rc cache/expire remote=/ withData=true + +### cache/stats: Get cache stats + +Show statistics for the cache remote. + +### core/bwlimit: Set the bandwidth limit. + +This sets the bandwidth limit to that passed in. + +Eg + + rclone rc core/bwlimit rate=1M + rclone rc core/bwlimit rate=off + +The format of the parameter is exactly the same as passed to --bwlimit +except only one bandwidth may be specified. + +### core/gc: Runs a garbage collection. + +This tells the go runtime to do a garbage collection run. It isn't +necessary to call this normally, but it can be useful for debugging +memory problems. + +### core/memstats: Returns the memory statistics + +This returns the memory statistics of the running program. What the values mean +are explained in the go docs: https://golang.org/pkg/runtime/#MemStats + +The most interesting values for most people are: + +* HeapAlloc: This is the amount of memory rclone is actually using +* HeapSys: This is the amount of memory rclone has obtained from the OS +* Sys: this is the total amount of memory requested from the OS + * It is virtual memory so may include unused memory + +### core/pid: Return PID of current process + +This returns PID of current process. +Useful for stopping rclone process. + +### core/stats: Returns stats about current transfers. + +This returns all available stats + + rclone rc core/stats + +Returns the following values: + +``` +{ + "speed": average speed in bytes/sec since start of the process, + "bytes": total transferred bytes since the start of the process, + "errors": number of errors, + "checks": number of checked files, + "transfers": number of transferred files, + "deletes" : number of deleted files, + "elapsedTime": time in seconds since the start of the process, + "lastError": last occurred error, + "transferring": an array of currently active file transfers: + [ + { + "bytes": total transferred bytes for this file, + "eta": estimated time in seconds until file transfer completion + "name": name of the file, + "percentage": progress of the file transfer in percent, + "speed": speed in bytes/sec, + "speedAvg": speed in bytes/sec as an exponentially weighted moving average, + "size": size of the file in bytes + } + ], + "checking": an array of names of currently active file checks + [] +} +``` +Values for "transferring", "checking" and "lastError" are only assigned if data is available. +The value for "eta" is null if an eta cannot be determined. + +### rc/error: This returns an error + +This returns an error with the input as part of its error string. +Useful for testing error handling. + +### rc/list: List all the registered remote control commands + +This lists all the registered remote control commands as a JSON map in +the commands response. + +### rc/noop: Echo the input to the output parameters + +This echoes the input parameters to the output parameters for testing +purposes. It can be used to check that rclone is still alive and to +check that parameter passing is working properly. + +### vfs/forget: Forget files or directories in the directory cache. + +This forgets the paths in the directory cache causing them to be +re-read from the remote when needed. + +If no paths are passed in then it will forget all the paths in the +directory cache. + + rclone rc vfs/forget + +Otherwise pass files or dirs in as file=path or dir=path. Any +parameter key starting with file will forget that file and any +starting with dir will forget that dir, eg + + rclone rc vfs/forget file=hello file2=goodbye dir=home/junk + +### vfs/refresh: Refresh the directory cache. + +This reads the directories for the specified paths and freshens the +directory cache. + +If no paths are passed in then it will refresh the root directory. + + rclone rc vfs/refresh + +Otherwise pass directories in as dir=path. Any parameter key +starting with dir will refresh that directory, eg + + rclone rc vfs/refresh dir=home/junk dir2=data/misc + +If the parameter recursive=true is given the whole directory tree +will get refreshed. This refresh will use --fast-list if enabled. + + + +## Accessing the remote control via HTTP + +Rclone implements a simple HTTP based protocol. + +Each endpoint takes an JSON object and returns a JSON object or an +error. The JSON objects are essentially a map of string names to +values. + +All calls must made using POST. + +The input objects can be supplied using URL parameters, POST +parameters or by supplying "Content-Type: application/json" and a JSON +blob in the body. There are examples of these below using `curl`. + +The response will be a JSON blob in the body of the response. This is +formatted to be reasonably human readable. + +If an error occurs then there will be an HTTP error status (usually +400) and the body of the response will contain a JSON encoded error +object. + +### Using POST with URL parameters only + +``` +curl -X POST 'http://localhost:5572/rc/noop/?potato=1&sausage=2' +``` + +Response + +``` +{ + "potato": "1", + "sausage": "2" +} +``` + +Here is what an error response looks like: + +``` +curl -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2' +``` + +``` +{ + "error": "arbitrary error on input map[potato:1 sausage:2]", + "input": { + "potato": "1", + "sausage": "2" + } +} +``` + +Note that curl doesn't return errors to the shell unless you use the `-f` option + +``` +$ curl -f -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2' +curl: (22) The requested URL returned error: 400 Bad Request +$ echo $? +22 +``` + +### Using POST with a form + +``` +curl --data "potato=1" --data "sausage=2" http://localhost:5572/rc/noop/ +``` + +Response + +``` +{ + "potato": "1", + "sausage": "2" +} +``` + +Note that you can combine these with URL parameters too with the POST +parameters taking precedence. + +``` +curl --data "potato=1" --data "sausage=2" "http://localhost:5572/rc/noop/?rutabaga=3&sausage=4" +``` + +Response + +``` +{ + "potato": "1", + "rutabaga": "3", + "sausage": "4" +} + +``` + +### Using POST with a JSON blob + +``` +curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' http://localhost:5572/rc/noop/ +``` + +response + +``` +{ + "password": "xyz", + "username": "xyz" +} +``` + +This can be combined with URL parameters too if required. The JSON +blob takes precedence. + +``` +curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' 'http://localhost:5572/rc/noop/?rutabaga=3&potato=4' +``` + +``` +{ + "potato": 2, + "rutabaga": "3", + "sausage": 1 +} +``` + +## Debugging rclone with pprof ## + +If you use the `--rc` flag this will also enable the use of the go +profiling tools on the same port. + +To use these, first [install go](https://golang.org/doc/install). + +Then (for example) to profile rclone's memory use you can run: + + go tool pprof -web http://localhost:5572/debug/pprof/heap + +This should open a page in your browser showing what is using what +memory. + +You can also use the `-text` flag to produce a textual summary + +``` +$ go tool pprof -text http://localhost:5572/debug/pprof/heap +Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total + flat flat% sum% cum cum% + 1024.03kB 66.62% 66.62% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.addDecoderNode + 513kB 33.38% 100% 513kB 33.38% net/http.newBufioWriterSize + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/cmd/all.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/cmd/serve.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/cmd/serve/restic.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init.0 + 0 0% 100% 1024.03kB 66.62% main.init + 0 0% 100% 513kB 33.38% net/http.(*conn).readRequest + 0 0% 100% 513kB 33.38% net/http.(*conn).serve + 0 0% 100% 1024.03kB 66.62% runtime.main +``` + +Possible profiles to look at: + + * Memory: `go tool pprof http://localhost:5572/debug/pprof/heap` + * 30-second CPU profile: `go tool pprof http://localhost:5572/debug/pprof/profile` + * 5-second execution trace: `wget http://localhost:5572/debug/pprof/trace?seconds=5` + +See the [net/http/pprof docs](https://golang.org/pkg/net/http/pprof/) +for more info on how to use the profiling and for a general overview +see [the Go team's blog post on profiling go programs](https://blog.golang.org/profiling-go-programs). + +The profiling hook is [zero overhead unless it is used](https://stackoverflow.com/q/26545159/164234). + +# Overview of cloud storage systems # + +Each cloud storage system is slightly different. Rclone attempts to +provide a unified interface to them, but some underlying differences +show through. + +## Features ## + +Here is an overview of the major features of each cloud storage system. + +| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | +| ---------------------------- |:-----------:|:-------:|:----------------:|:---------------:|:---------:| +| Amazon Drive | MD5 | No | Yes | No | R | +| Amazon S3 | MD5 | Yes | No | No | R/W | +| Backblaze B2 | SHA1 | Yes | No | No | R/W | +| Box | SHA1 | Yes | Yes | No | - | +| Dropbox | DBHASH † | Yes | Yes | No | - | +| FTP | - | No | No | No | - | +| Google Cloud Storage | MD5 | Yes | No | No | R/W | +| Google Drive | MD5 | Yes | No | Yes | R/W | +| HTTP | - | No | No | No | R | +| Hubic | MD5 | Yes | No | No | R/W | +| Jottacloud | MD5 | Yes | Yes | No | R/W | +| Mega | - | No | No | Yes | - | +| Microsoft Azure Blob Storage | MD5 | Yes | No | No | R/W | +| Microsoft OneDrive | SHA1 ‡‡ | Yes | Yes | No | R | +| OpenDrive | MD5 | Yes | Yes | No | - | +| Openstack Swift | MD5 | Yes | No | No | R/W | +| pCloud | MD5, SHA1 | Yes | No | No | W | +| QingStor | MD5 | No | No | No | R/W | +| SFTP | MD5, SHA1 ‡ | Yes | Depends | No | - | +| WebDAV | - | Yes †† | Depends | No | - | +| Yandex Disk | MD5 | Yes | No | No | R/W | +| The local filesystem | All | Yes | Depends | No | - | + +### Hash ### + +The cloud storage system supports various hash types of the objects. +The hashes are used when transferring data as an integrity check and +can be specifically used with the `--checksum` flag in syncs and in +the `check` command. + +To use the verify checksums when transferring between cloud storage +systems they must support a common hash type. + +† Note that Dropbox supports [its own custom +hash](https://www.dropbox.com/developers/reference/content-hash). +This is an SHA256 sum of all the 4MB block SHA256s. + +‡ SFTP supports checksums if the same login has shell access and `md5sum` +or `sha1sum` as well as `echo` are in the remote's PATH. + +†† WebDAV supports modtimes when used with Owncloud and Nextcloud only. + +‡‡ Microsoft OneDrive Personal supports SHA1 hashes, whereas OneDrive +for business and SharePoint server support Microsoft's own +[QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash). + +### ModTime ### + +The cloud storage system supports setting modification times on +objects. If it does then this enables a using the modification times +as part of the sync. If not then only the size will be checked by +default, though the MD5SUM can be checked with the `--checksum` flag. + +All cloud storage systems support some kind of date on the object and +these will be set when transferring from the cloud storage system. + +### Case Insensitive ### + +If a cloud storage systems is case sensitive then it is possible to +have two files which differ only in case, eg `file.txt` and +`FILE.txt`. If a cloud storage system is case insensitive then that +isn't possible. + +This can cause problems when syncing between a case insensitive +system and a case sensitive system. The symptom of this is that no +matter how many times you run the sync it never completes fully. + +The local filesystem and SFTP may or may not be case sensitive +depending on OS. + + * Windows - usually case insensitive, though case is preserved + * OSX - usually case insensitive, though it is possible to format case sensitive + * Linux - usually case sensitive, but there are case insensitive file systems (eg FAT formatted USB keys) + +Most of the time this doesn't cause any problems as people tend to +avoid files whose name differs only by case even on case sensitive +systems. + +### Duplicate files ### + +If a cloud storage system allows duplicate files then it can have two +objects with the same name. + +This confuses rclone greatly when syncing - use the `rclone dedupe` +command to rename or remove duplicates. + +### MIME Type ### + +MIME types (also known as media types) classify types of documents +using a simple text classification, eg `text/html` or +`application/pdf`. + +Some cloud storage systems support reading (`R`) the MIME type of +objects and some support writing (`W`) the MIME type of objects. + +The MIME type can be important if you are serving files directly to +HTTP from the storage system. + +If you are copying from a remote which supports reading (`R`) to a +remote which supports writing (`W`) then rclone will preserve the MIME +types. Otherwise they will be guessed from the extension, or the +remote itself may assign the MIME type. + +## Optional Features ## + +All the remotes support a basic set of features, but there are some +optional features supported by some remotes used to make some +operations more efficient. + +| Name | Purge | Copy | Move | DirMove | CleanUp | ListR | StreamUpload | LinkSharing | About | +| ---------------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:|:------------:|:------------:|:-----:| +| Amazon Drive | Yes | No | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| Amazon S3 | No | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| Backblaze B2 | No | No | No | No | Yes | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| Box | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| Dropbox | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | Yes | Yes | Yes | +| FTP | No | No | Yes | Yes | No | No | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | +| HTTP | No | No | No | No | No | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| Hubic | Yes † | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes | +| Jottacloud | Yes | Yes | Yes | Yes | No | No | No | No | No | +| Mega | Yes | No | Yes | Yes | No | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes | +| Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| Microsoft OneDrive | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes | +| OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | +| Openstack Swift | Yes † | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes | +| pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes | +| QingStor | No | Yes | No | No | No | Yes | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| SFTP | No | No | Yes | Yes | No | No | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| Yandex Disk | Yes | No | No | No | Yes | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| The local filesystem | Yes | No | Yes | Yes | No | No | Yes | No | Yes | + +### Purge ### + +This deletes a directory quicker than just deleting all the files in +the directory. + +† Note Swift and Hubic implement this in order to delete directory +markers but they don't actually have a quicker way of deleting files +other than deleting them individually. + +‡ StreamUpload is not supported with Nextcloud + +### Copy ### + +Used when copying an object to and from the same remote. This known +as a server side copy so you can copy a file without downloading it +and uploading it again. It is used if you use `rclone copy` or +`rclone move` if the remote doesn't support `Move` directly. + +If the server doesn't support `Copy` directly then for copy operations +the file is downloaded then re-uploaded. + +### Move ### + +Used when moving/renaming an object on the same remote. This is known +as a server side move of a file. This is used in `rclone move` if the +server doesn't support `DirMove`. + +If the server isn't capable of `Move` then rclone simulates it with +`Copy` then delete. If the server doesn't support `Copy` then rclone +will download the file and re-upload it. + +### DirMove ### + +This is used to implement `rclone move` to move a directory if +possible. If it isn't then it will use `Move` on each file (which +falls back to `Copy` then download and upload - see `Move` section). + +### CleanUp ### + +This is used for emptying the trash for a remote by `rclone cleanup`. + +If the server can't do `CleanUp` then `rclone cleanup` will return an +error. + +### ListR ### + +The remote supports a recursive list to list all the contents beneath +a directory quickly. This enables the `--fast-list` flag to work. +See the [rclone docs](/docs/#fast-list) for more details. + +### StreamUpload ### + +Some remotes allow files to be uploaded without knowing the file size +in advance. This allows certain operations to work without spooling the +file to local disk first, e.g. `rclone rcat`. + +### LinkSharing ### + +Sets the necessary permissions on a file or folder and prints a link +that allows others to access them, even if they don't have an account +on the particular cloud provider. + +### About ### + +This is used to fetch quota information from the remote, like bytes +used/free/quota and bytes used in the trash. + +If the server can't do `About` then `rclone about` will return an +error. + +Alias +----------------------------------------- + +The `alias` remote provides a new name for another remote. + +Paths may be as deep as required or a local path, +eg `remote:directory/subdirectory` or `/directory/subdirectory`. + +During the initial setup with `rclone config` you will specify the target +remote. The target remote can either be a local path or another remote. + +Subfolders can be used in target remote. Asume a alias remote named `backup` +with the target `mydrive:private/backup`. Invoking `rclone mkdir backup:desktop` +is exactly the same as invoking `rclone mkdir mydrive:private/backup/desktop`. + +There will be no special handling of paths containing `..` segments. +Invoking `rclone mkdir backup:../desktop` is exactly the same as invoking +`rclone mkdir mydrive:private/backup/../desktop`. +The empty path is not allowed as a remote. To alias the current directory +use `.` instead. + +Here is an example of how to make a alias called `remote` for local folder. +First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" + 2 / Amazon Drive + \ "amazon cloud drive" + 3 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 4 / Backblaze B2 + \ "b2" + 5 / Box + \ "box" + 6 / Cache a remote + \ "cache" + 7 / Dropbox + \ "dropbox" + 8 / Encrypt/Decrypt a remote + \ "crypt" + 9 / FTP Connection + \ "ftp" +10 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" +11 / Google Drive + \ "drive" +12 / Hubic + \ "hubic" +13 / Local Disk + \ "local" +14 / Microsoft Azure Blob Storage + \ "azureblob" +15 / Microsoft OneDrive + \ "onedrive" +16 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +17 / Pcloud + \ "pcloud" +18 / QingCloud Object Storage + \ "qingstor" +19 / SSH/SFTP Connection + \ "sftp" +20 / Webdav + \ "webdav" +21 / Yandex Disk + \ "yandex" +22 / http Connection + \ "http" +Storage> 1 +Remote or path to alias. +Can be "myremote:path/to/dir", "myremote:bucket", "myremote:" or "/local/path". +remote> /mnt/storage/backup +Remote config +-------------------- +[remote] +remote = /mnt/storage/backup +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +Current remotes: + +Name Type +==== ==== +remote alias + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> q +``` + +Once configured you can then use `rclone` like this, + +List directories in top level in `/mnt/storage/backup` + + rclone lsd remote: + +List all the files in `/mnt/storage/backup` + + rclone ls remote: + +Copy another local directory to the alias directory called source + + rclone copy /home/source remote:source + +Amazon Drive +----------------------------------------- + +Amazon Drive, formerly known as Amazon Cloud Drive, is a cloud storage +service run by Amazon for consumers. + +## Status + +**Important:** rclone supports Amazon Drive only if you have your own +set of API keys. Unfortunately the [Amazon Drive developer +program](https://developer.amazon.com/amazon-drive) is now closed to +new entries so if you don't already have your own set of keys you will +not be able to use rclone with Amazon Drive. + +For the history on why rclone no longer has a set of Amazon Drive API +keys see [the forum](https://forum.rclone.org/t/rclone-has-been-banned-from-amazon-drive/2314). + +If you happen to know anyone who works at Amazon then please ask them +to re-instate rclone into the Amazon Drive developer program - thanks! + +## Setup + +The initial setup for Amazon Drive involves getting a token from +Amazon which you need to do in your browser. `rclone config` walks +you through it. + +The configuration process for Amazon Drive may involve using an [oauth +proxy](https://github.com/ncw/oauthproxy). This is used to keep the +Amazon credentials out of the source code. The proxy runs in Google's +very secure App Engine environment and doesn't store any credentials +which pass through it. + +Since rclone doesn't currently have its own Amazon Drive credentials +so you will either need to have your own `client_id` and +`client_secret` with Amazon Drive, or use a a third party ouath proxy +in which case you will need to enter `client_id`, `client_secret`, +`auth_url` and `token_url`. + +Note also if you are not using Amazon's `auth_url` and `token_url`, +(ie you filled in something for those) then if setting up on a remote +machine you can only use the [copying the config method of +configuration](https://rclone.org/remote_setup/#configuring-by-copying-the-config-file) +- `rclone authorize` will not work. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +n/r/c/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / FTP Connection + \ "ftp" + 7 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 8 / Google Drive + \ "drive" + 9 / Hubic + \ "hubic" +10 / Local Disk + \ "local" +11 / Microsoft OneDrive + \ "onedrive" +12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +13 / SSH/SFTP Connection + \ "sftp" +14 / Yandex Disk + \ "yandex" +Storage> 1 +Amazon Application Client Id - required. +client_id> your client ID goes here +Amazon Application Client Secret - required. +client_secret> your client secret goes here +Auth server URL - leave blank to use Amazon's. +auth_url> Optional auth URL +Token server url - leave blank to use Amazon's. +token_url> Optional token URL +Remote config +Make sure your Redirect URL is set to "http://127.0.0.1:53682/" in your custom config. +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +client_id = your client ID goes here +client_secret = your client secret goes here +auth_url = Optional auth URL +token_url = Optional token URL +token = {"access_token":"xxxxxxxxxxxxxxxxxxxxxxx","token_type":"bearer","refresh_token":"xxxxxxxxxxxxxxxxxx","expiry":"2015-09-06T16:07:39.658438471+01:00"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Amazon. This only runs from the moment it +opens your browser to the moment you get back the verification +code. This is on `http://127.0.0.1:53682/` and this it may require +you to unblock it temporarily if you are running a host firewall. + +Once configured you can then use `rclone` like this, + +List directories in top level of your Amazon Drive + + rclone lsd remote: + +List all the files in your Amazon Drive + + rclone ls remote: + +To copy a local directory to an Amazon Drive directory called backup + + rclone copy /home/source remote:backup + +### Modified time and MD5SUMs ### + +Amazon Drive doesn't allow modification times to be changed via +the API so these won't be accurate or used for syncing. + +It does store MD5SUMs so for a more accurate sync, you can use the +`--checksum` flag. + +### Deleting files ### + +Any files you delete with rclone will end up in the trash. Amazon +don't provide an API to permanently delete files, nor to empty the +trash, so you will have to do that with one of Amazon's apps or via +the Amazon Drive website. As of November 17, 2016, files are +automatically deleted by Amazon from the trash after 30 days. + +### Using with non `.com` Amazon accounts ### + +Let's say you usually use `amazon.co.uk`. When you authenticate with +rclone it will take you to an `amazon.com` page to log in. Your +`amazon.co.uk` email and password should work here just fine. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --acd-templink-threshold=SIZE #### + +Files this size or more will be downloaded via their `tempLink`. This +is to work around a problem with Amazon Drive which blocks downloads +of files bigger than about 10GB. The default for this is 9GB which +shouldn't need to be changed. + +To download files above this threshold, rclone requests a `tempLink` +which downloads the file through a temporary URL directly from the +underlying S3 storage. + +#### --acd-upload-wait-per-gb=TIME #### + +Sometimes Amazon Drive gives an error when a file has been fully +uploaded but the file appears anyway after a little while. This +happens sometimes for files over 1GB in size and nearly every time for +files bigger than 10GB. This parameter controls the time rclone waits +for the file to appear. + +The default value for this parameter is 3 minutes per GB, so by +default it will wait 3 minutes for every GB uploaded to see if the +file appears. + +You can disable this feature by setting it to 0. This may cause +conflict errors as rclone retries the failed upload but the file will +most likely appear correctly eventually. + +These values were determined empirically by observing lots of uploads +of big files for a range of file sizes. + +Upload with the `-v` flag to see more info about what rclone is doing +in this situation. + +### Limitations ### + +Note that Amazon Drive is case insensitive so you can't have a +file called "Hello.doc" and one called "hello.doc". + +Amazon Drive has rate limiting so you may notice errors in the +sync (429 errors). rclone will automatically retry the sync up to 3 +times by default (see `--retries` flag) which should hopefully work +around this problem. + +Amazon Drive has an internal limit of file sizes that can be uploaded +to the service. This limit is not officially published, but all files +larger than this will fail. + +At the time of writing (Jan 2016) is in the area of 50GB per file. +This means that larger files are likely to fail. + +Unfortunately there is no way for rclone to see that this failure is +because of file size, so it will retry the operation, as any other +failure. To avoid this problem, use `--max-size 50000M` option to limit +the maximum size of uploaded files. Note that `--max-size` does not split +files into segments, it only ignores files over this size. + +Amazon S3 Storage Providers +-------------------------------------------------------- + +The S3 backend can be used with a number of different providers: + +* AWS S3 +* Ceph +* DigitalOcean Spaces +* Dreamhost +* IBM COS S3 +* Minio +* Wasabi + +Paths are specified as `remote:bucket` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, eg `remote:bucket/path/to/dir`. + +Once you have made a remote (see the provider specific section above) +you can use it like this: + +See all buckets + + rclone lsd remote: + +Make a new bucket + + rclone mkdir remote:bucket + +List the contents of a bucket + + rclone ls remote:bucket + +Sync `/home/local/directory` to the remote bucket, deleting any excess +files in the bucket. + + rclone sync /home/local/directory remote:bucket + +## AWS S3 {#amazon-s3} + +Here is an example of making an s3 configuration. First run + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" + 2 / Amazon Drive + \ "amazon cloud drive" + 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio) + \ "s3" + 4 / Backblaze B2 + \ "b2" +[snip] +23 / http Connection + \ "http" +Storage> s3 +Choose your S3 provider. +Choose a number from below, or type in your own value + 1 / Amazon Web Services (AWS) S3 + \ "AWS" + 2 / Ceph Object Storage + \ "Ceph" + 3 / Digital Ocean Spaces + \ "DigitalOcean" + 4 / Dreamhost DreamObjects + \ "Dreamhost" + 5 / IBM COS S3 + \ "IBMCOS" + 6 / Minio Object Storage + \ "Minio" + 7 / Wasabi Object Storage + \ "Wasabi" + 8 / Any other S3 compatible provider + \ "Other" +provider> 1 +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" +env_auth> 1 +AWS Access Key ID - leave blank for anonymous access or runtime credentials. +access_key_id> XXX +AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. +secret_access_key> YYY +Region to connect to. +Choose a number from below, or type in your own value + / The default endpoint - a good choice if you are unsure. + 1 | US Region, Northern Virginia or Pacific Northwest. + | Leave location constraint empty. + \ "us-east-1" + / US East (Ohio) Region + 2 | Needs location constraint us-east-2. + \ "us-east-2" + / US West (Oregon) Region + 3 | Needs location constraint us-west-2. + \ "us-west-2" + / US West (Northern California) Region + 4 | Needs location constraint us-west-1. + \ "us-west-1" + / Canada (Central) Region + 5 | Needs location constraint ca-central-1. + \ "ca-central-1" + / EU (Ireland) Region + 6 | Needs location constraint EU or eu-west-1. + \ "eu-west-1" + / EU (London) Region + 7 | Needs location constraint eu-west-2. + \ "eu-west-2" + / EU (Frankfurt) Region + 8 | Needs location constraint eu-central-1. + \ "eu-central-1" + / Asia Pacific (Singapore) Region + 9 | Needs location constraint ap-southeast-1. + \ "ap-southeast-1" + / Asia Pacific (Sydney) Region +10 | Needs location constraint ap-southeast-2. + \ "ap-southeast-2" + / Asia Pacific (Tokyo) Region +11 | Needs location constraint ap-northeast-1. + \ "ap-northeast-1" + / Asia Pacific (Seoul) +12 | Needs location constraint ap-northeast-2. + \ "ap-northeast-2" + / Asia Pacific (Mumbai) +13 | Needs location constraint ap-south-1. + \ "ap-south-1" + / South America (Sao Paulo) Region +14 | Needs location constraint sa-east-1. + \ "sa-east-1" +region> 1 +Endpoint for S3 API. +Leave blank if using AWS to use the default endpoint for the region. +endpoint> +Location constraint - must be set to match the Region. Used when creating buckets only. +Choose a number from below, or type in your own value + 1 / Empty for US Region, Northern Virginia or Pacific Northwest. + \ "" + 2 / US East (Ohio) Region. + \ "us-east-2" + 3 / US West (Oregon) Region. + \ "us-west-2" + 4 / US West (Northern California) Region. + \ "us-west-1" + 5 / Canada (Central) Region. + \ "ca-central-1" + 6 / EU (Ireland) Region. + \ "eu-west-1" + 7 / EU (London) Region. + \ "eu-west-2" + 8 / EU Region. + \ "EU" + 9 / Asia Pacific (Singapore) Region. + \ "ap-southeast-1" +10 / Asia Pacific (Sydney) Region. + \ "ap-southeast-2" +11 / Asia Pacific (Tokyo) Region. + \ "ap-northeast-1" +12 / Asia Pacific (Seoul) + \ "ap-northeast-2" +13 / Asia Pacific (Mumbai) + \ "ap-south-1" +14 / South America (Sao Paulo) Region. + \ "sa-east-1" +location_constraint> 1 +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" + 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access. + \ "public-read" + / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. + 3 | Granting this on a bucket is generally not recommended. + \ "public-read-write" + 4 / Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. + \ "authenticated-read" + / Object owner gets FULL_CONTROL. Bucket owner gets READ access. + 5 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ "bucket-owner-read" + / Both the object owner and the bucket owner get FULL_CONTROL over the object. + 6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ "bucket-owner-full-control" +acl> 1 +The server-side encryption algorithm used when storing this object in S3. +Choose a number from below, or type in your own value + 1 / None + \ "" + 2 / AES256 + \ "AES256" +server_side_encryption> 1 +The storage class to use when storing objects in S3. +Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Standard storage class + \ "STANDARD" + 3 / Reduced redundancy storage class + \ "REDUCED_REDUNDANCY" + 4 / Standard Infrequent Access storage class + \ "STANDARD_IA" + 5 / One Zone Infrequent Access storage class + \ "ONEZONE_IA" +storage_class> 1 +Remote config +-------------------- +[remote] +type = s3 +provider = AWS +env_auth = false +access_key_id = XXX +secret_access_key = YYY +region = us-east-1 +endpoint = +location_constraint = +acl = private +server_side_encryption = +storage_class = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> +``` + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +### --update and --use-server-modtime ### + +As noted below, the modified time is stored on metadata on the object. It is +used by default for all operations that require checking the time a file was +last updated. It allows rclone to treat the remote more like a true filesystem, +but it is inefficient because it requires an extra API call to retrieve the +metadata. + +For many operations, the time the object was last uploaded to the remote is +sufficient to determine if it is "dirty". By using `--update` along with +`--use-server-modtime`, you can avoid the extra API call and simply upload +files whose local modtime is newer than the time it was last uploaded. + +### Modified time ### + +The modified time is stored as metadata on the object as +`X-Amz-Meta-Mtime` as floating point since the epoch accurate to 1 ns. + +### Multipart uploads ### + +rclone supports multipart uploads with S3 which means that it can +upload files bigger than 5GB. Note that files uploaded *both* with +multipart upload *and* through crypt remotes do not have MD5 sums. + +### Buckets and Regions ### + +With Amazon S3 you can list buckets (`rclone lsd`) using any region, +but you can only access the content of a bucket from the region it was +created in. If you attempt to access a bucket from the wrong region, +you will get an error, `incorrect region, the bucket is not in 'XXX' +region`. + +### Authentication ### + +There are a number of ways to supply `rclone` with a set of AWS +credentials, with and without using the environment. + +The different authentication methods are tried in this order: + + - Directly in the rclone configuration file (`env_auth = false` in the config file): + - `access_key_id` and `secret_access_key` are required. + - `session_token` can be optionally set when using AWS STS. + - Runtime configuration (`env_auth = true` in the config file): + - Export the following environment variables before running `rclone`: + - Access Key ID: `AWS_ACCESS_KEY_ID` or `AWS_ACCESS_KEY` + - Secret Access Key: `AWS_SECRET_ACCESS_KEY` or `AWS_SECRET_KEY` + - Session Token: `AWS_SESSION_TOKEN` (optional) + - Or, use a [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-multiple-profiles.html): + - Profile files are standard files used by AWS CLI tools + - By default it will use the profile in your home directory (eg `~/.aws/credentials` on unix based systems) file and the "default" profile, to change set these environment variables: + - `AWS_SHARED_CREDENTIALS_FILE` to control which file. + - `AWS_PROFILE` to control which profile to use. + - Or, run `rclone` in an ECS task with an IAM role (AWS only). + - Or, run `rclone` on an EC2 instance with an IAM role (AWS only). + +If none of these option actually end up providing `rclone` with AWS +credentials then S3 interaction will be non-authenticated (see below). + +### S3 Permissions ### + +When using the `sync` subcommand of `rclone` the following minimum +permissions are required to be available on the bucket being written to: + +* `ListBucket` +* `DeleteObject` +* `GetObject` +* `PutObject` +* `PutObjectACL` + +Example policy: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::USER_SID:user/USER_NAME" + }, + "Action": [ + "s3:ListBucket", + "s3:DeleteObject", + "s3:GetObject", + "s3:PutObject", + "s3:PutObjectAcl" + ], + "Resource": [ + "arn:aws:s3:::BUCKET_NAME/*", + "arn:aws:s3:::BUCKET_NAME" + ] + } + ] +} +``` + +Notes on above: + +1. This is a policy that can be used when creating bucket. It assumes + that `USER_NAME` has been created. +2. The Resource entry must include both resource ARNs, as one implies + the bucket and the other implies the bucket's objects. + +For reference, [here's an Ansible script](https://gist.github.com/ebridges/ebfc9042dd7c756cd101cfa807b7ae2b) +that will generate one or more buckets that will work with `rclone sync`. + +### Key Management System (KMS) ### + +If you are using server side encryption with KMS then you will find +you can't transfer small objects. As a work-around you can use the +`--ignore-checksum` flag. + +A proper fix is being worked on in [issue #1824](https://github.com/ncw/rclone/issues/1824). + +### Glacier ### + +You can transition objects to glacier storage using a [lifecycle policy](http://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-lifecycle.html). +The bucket can still be synced or copied into normally, but if rclone +tries to access the data you will see an error like below. + + 2017/09/11 19:07:43 Failed to sync: failed to open source object: Object in GLACIER, restore first: path/to/file + +In this case you need to [restore](http://docs.aws.amazon.com/AmazonS3/latest/user-guide/restore-archived-objects.html) +the object(s) in question before using rclone. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --s3-acl=STRING #### + +Canned ACL used when creating buckets and/or storing objects in S3. + +For more info visit the [canned ACL docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl). + +#### --s3-storage-class=STRING #### + +Storage class to upload new objects with. + +Available options include: + + - STANDARD - default storage class + - STANDARD_IA - for less frequently accessed data (e.g backups) + - ONEZONE_IA - for storing data in only one Availability Zone + - REDUCED_REDUNDANCY (only for noncritical, reproducible data, has lower redundancy) + +#### --s3-chunk-size=SIZE #### + +Any files larger than this will be uploaded in chunks of this +size. The default is 5MB. The minimum is 5MB. + +Note that 2 chunks of this size are buffered in memory per transfer. + +If you are transferring large files over high speed links and you have +enough memory, then increasing this will speed up the transfers. + +#### --s3-force-path-style=BOOL #### + +If this is true (the default) then rclone will use path style access, +if false then rclone will use virtual path style. See [the AWS S3 +docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro) +for more info. + +Some providers (eg Aliyun OSS or Netease COS) require this set to +`false`. It can also be set in the config in the advanced section. + +#### --s3-upload-concurrency #### + +Number of chunks of the same file that are uploaded concurrently. +Default is 2. + +If you are uploading small amount of large file over high speed link +and these uploads do not fully utilize your bandwidth, then increasing +this may help to speed up the transfers. + +### Anonymous access to public buckets ### + +If you want to use rclone to access a public bucket, configure with a +blank `access_key_id` and `secret_access_key`. Your config should end +up looking like this: + +``` +[anons3] +type = s3 +provider = AWS +env_auth = false +access_key_id = +secret_access_key = +region = us-east-1 +endpoint = +location_constraint = +acl = private +server_side_encryption = +storage_class = +``` + +Then use it as normal with the name of the public bucket, eg + + rclone lsd anons3:1000genomes + +You will be able to list and copy data but not upload it. + +### Ceph ### + +[Ceph](https://ceph.com/) is an open source unified, distributed +storage system designed for excellent performance, reliability and +scalability. It has an S3 compatible object storage interface. + +To use rclone with Ceph, configure as above but leave the region blank +and set the endpoint. You should end up with something like this in +your config: + + +``` +[ceph] +type = s3 +provider = Ceph +env_auth = false +access_key_id = XXX +secret_access_key = YYY +region = +endpoint = https://ceph.endpoint.example.com +location_constraint = +acl = +server_side_encryption = +storage_class = +``` + +Note also that Ceph sometimes puts `/` in the passwords it gives +users. If you read the secret access key using the command line tools +you will get a JSON blob with the `/` escaped as `\/`. Make sure you +only write `/` in the secret access key. + +Eg the dump from Ceph looks something like this (irrelevant keys +removed). + +``` +{ + "user_id": "xxx", + "display_name": "xxxx", + "keys": [ + { + "user": "xxx", + "access_key": "xxxxxx", + "secret_key": "xxxxxx\/xxxx" + } + ], +} +``` + +Because this is a json dump, it is encoding the `/` as `\/`, so if you +use the secret key as `xxxxxx/xxxx` it will work fine. + +### Dreamhost ### + +Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is +an object storage system based on CEPH. + +To use rclone with Dreamhost, configure as above but leave the region blank +and set the endpoint. You should end up with something like this in +your config: + +``` +[dreamobjects] +type = s3 +provider = DreamHost +env_auth = false +access_key_id = your_access_key +secret_access_key = your_secret_key +region = +endpoint = objects-us-west-1.dream.io +location_constraint = +acl = private +server_side_encryption = +storage_class = +``` + +### DigitalOcean Spaces ### + +[Spaces](https://www.digitalocean.com/products/object-storage/) is an [S3-interoperable](https://developers.digitalocean.com/documentation/spaces/) object storage service from cloud provider DigitalOcean. + +To connect to DigitalOcean Spaces you will need an access key and secret key. These can be retrieved on the "[Applications & API](https://cloud.digitalocean.com/settings/api/tokens)" page of the DigitalOcean control panel. They will be needed when promted by `rclone config` for your `access_key_id` and `secret_access_key`. + +When prompted for a `region` or `location_constraint`, press enter to use the default value. The region must be included in the `endpoint` setting (e.g. `nyc3.digitaloceanspaces.com`). The defualt values can be used for other settings. + +Going through the whole process of creating a new remote by running `rclone config`, each prompt should be answered as shown below: + +``` +Storage> s3 +env_auth> 1 +access_key_id> YOUR_ACCESS_KEY +secret_access_key> YOUR_SECRET_KEY +region> +endpoint> nyc3.digitaloceanspaces.com +location_constraint> +acl> +storage_class> +``` + +The resulting configuration file should look like: + +``` +[spaces] +type = s3 +provider = DigitalOcean +env_auth = false +access_key_id = YOUR_ACCESS_KEY +secret_access_key = YOUR_SECRET_KEY +region = +endpoint = nyc3.digitaloceanspaces.com +location_constraint = +acl = +server_side_encryption = +storage_class = +``` + +Once configured, you can create a new Space and begin copying files. For example: + +``` +rclone mkdir spaces:my-new-space +rclone copy /path/to/files spaces:my-new-space +``` + +### IBM COS (S3) ### + +Information stored with IBM Cloud Object Storage is encrypted and dispersed across multiple geographic locations, and accessed through an implementation of the S3 API. This service makes use of the distributed storage technologies provided by IBM’s Cloud Object Storage System (formerly Cleversafe). For more information visit: (http://www.ibm.com/cloud/object-storage) + +To configure access to IBM COS S3, follow the steps below: + +1. Run rclone config and select n for a new remote. +``` + 2018/02/14 14:13:11 NOTICE: Config file "C:\\Users\\a\\.config\\rclone\\rclone.conf" not found - using defaults + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n +``` + +2. Enter the name for the configuration +``` + name> +``` + +3. Select "s3" storage. +``` +Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" + 2 / Amazon Drive + \ "amazon cloud drive" + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, Minio, IBM COS) + \ "s3" + 4 / Backblaze B2 + \ "b2" +[snip] + 23 / http Connection + \ "http" +Storage> 3 +``` + +4. Select IBM COS as the S3 Storage Provider. +``` +Choose the S3 provider. +Choose a number from below, or type in your own value + 1 / Choose this option to configure Storage to AWS S3 + \ "AWS" + 2 / Choose this option to configure Storage to Ceph Systems + \ "Ceph" + 3 / Choose this option to configure Storage to Dreamhost + \ "Dreamhost" + 4 / Choose this option to the configure Storage to IBM COS S3 + \ "IBMCOS" + 5 / Choose this option to the configure Storage to Minio + \ "Minio" + Provider>4 +``` + +5. Enter the Access Key and Secret. +``` + AWS Access Key ID - leave blank for anonymous access or runtime credentials. + access_key_id> <> + AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. + secret_access_key> <> +``` + +6. Specify the endpoint for IBM COS. For Public IBM COS, choose from the option below. For On Premise IBM COS, enter an enpoint address. +``` + Endpoint for IBM COS S3 API. + Specify if using an IBM COS On Premise. + Choose a number from below, or type in your own value + 1 / US Cross Region Endpoint + \ "s3-api.us-geo.objectstorage.softlayer.net" + 2 / US Cross Region Dallas Endpoint + \ "s3-api.dal.us-geo.objectstorage.softlayer.net" + 3 / US Cross Region Washington DC Endpoint + \ "s3-api.wdc-us-geo.objectstorage.softlayer.net" + 4 / US Cross Region San Jose Endpoint + \ "s3-api.sjc-us-geo.objectstorage.softlayer.net" + 5 / US Cross Region Private Endpoint + \ "s3-api.us-geo.objectstorage.service.networklayer.com" + 6 / US Cross Region Dallas Private Endpoint + \ "s3-api.dal-us-geo.objectstorage.service.networklayer.com" + 7 / US Cross Region Washington DC Private Endpoint + \ "s3-api.wdc-us-geo.objectstorage.service.networklayer.com" + 8 / US Cross Region San Jose Private Endpoint + \ "s3-api.sjc-us-geo.objectstorage.service.networklayer.com" + 9 / US Region East Endpoint + \ "s3.us-east.objectstorage.softlayer.net" + 10 / US Region East Private Endpoint + \ "s3.us-east.objectstorage.service.networklayer.com" + 11 / US Region South Endpoint +[snip] + 34 / Toronto Single Site Private Endpoint + \ "s3.tor01.objectstorage.service.networklayer.com" + endpoint>1 +``` + + +7. Specify a IBM COS Location Constraint. The location constraint must match endpoint when using IBM Cloud Public. For on-prem COS, do not make a selection from this list, hit enter +``` + 1 / US Cross Region Standard + \ "us-standard" + 2 / US Cross Region Vault + \ "us-vault" + 3 / US Cross Region Cold + \ "us-cold" + 4 / US Cross Region Flex + \ "us-flex" + 5 / US East Region Standard + \ "us-east-standard" + 6 / US East Region Vault + \ "us-east-vault" + 7 / US East Region Cold + \ "us-east-cold" + 8 / US East Region Flex + \ "us-east-flex" + 9 / US South Region Standard + \ "us-south-standard" + 10 / US South Region Vault + \ "us-south-vault" +[snip] + 32 / Toronto Flex + \ "tor01-flex" +location_constraint>1 +``` + +9. Specify a canned ACL. IBM Cloud (Strorage) supports "public-read" and "private". IBM Cloud(Infra) supports all the canned ACLs. On-Premise COS supports all the canned ACLs. +``` +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise COS + \ "private" + 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access. This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise IBM COS + \ "public-read" + 3 / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. This acl is available on IBM Cloud (Infra), On-Premise IBM COS + \ "public-read-write" + 4 / Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS + \ "authenticated-read" +acl> 1 +``` + + +12. Review the displayed configuration and accept to save the "remote" then quit. The config file should look like this +``` + [xxx] + type = s3 + Provider = IBMCOS + access_key_id = xxx + secret_access_key = yyy + endpoint = s3-api.us-geo.objectstorage.softlayer.net + location_constraint = us-standard + acl = private +``` + +13. Execute rclone commands +``` + 1) Create a bucket. + rclone mkdir IBM-COS-XREGION:newbucket + 2) List available buckets. + rclone lsd IBM-COS-XREGION: + -1 2017-11-08 21:16:22 -1 test + -1 2018-02-14 20:16:39 -1 newbucket + 3) List contents of a bucket. + rclone ls IBM-COS-XREGION:newbucket + 18685952 test.exe + 4) Copy a file from local to remote. + rclone copy /Users/file.txt IBM-COS-XREGION:newbucket + 5) Copy a file from remote to local. + rclone copy IBM-COS-XREGION:newbucket/file.txt . + 6) Delete a file on remote. + rclone delete IBM-COS-XREGION:newbucket/file.txt +``` + +### Minio ### + +[Minio](https://minio.io/) is an object storage server built for cloud application developers and devops. + +It is very easy to install and provides an S3 compatible server which can be used by rclone. + +To use it, install Minio following the instructions [here](https://docs.minio.io/docs/minio-quickstart-guide). + +When it configures itself Minio will print something like this + +``` +Endpoint: http://192.168.1.106:9000 http://172.23.0.1:9000 +AccessKey: USWUXHGYZQYFYFFIT3RE +SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 +Region: us-east-1 +SQS ARNs: arn:minio:sqs:us-east-1:1:redis arn:minio:sqs:us-east-1:2:redis + +Browser Access: + http://192.168.1.106:9000 http://172.23.0.1:9000 + +Command-line Access: https://docs.minio.io/docs/minio-client-quickstart-guide + $ mc config host add myminio http://192.168.1.106:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 + +Object API (Amazon S3 compatible): + Go: https://docs.minio.io/docs/golang-client-quickstart-guide + Java: https://docs.minio.io/docs/java-client-quickstart-guide + Python: https://docs.minio.io/docs/python-client-quickstart-guide + JavaScript: https://docs.minio.io/docs/javascript-client-quickstart-guide + .NET: https://docs.minio.io/docs/dotnet-client-quickstart-guide + +Drive Capacity: 26 GiB Free, 165 GiB Total +``` + +These details need to go into `rclone config` like this. Note that it +is important to put the region in as stated above. + +``` +env_auth> 1 +access_key_id> USWUXHGYZQYFYFFIT3RE +secret_access_key> MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 +region> us-east-1 +endpoint> http://192.168.1.106:9000 +location_constraint> +server_side_encryption> +``` + +Which makes the config file look like this + +``` +[minio] +type = s3 +provider = Minio +env_auth = false +access_key_id = USWUXHGYZQYFYFFIT3RE +secret_access_key = MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 +region = us-east-1 +endpoint = http://192.168.1.106:9000 +location_constraint = +server_side_encryption = +``` + +So once set up, for example to copy files into a bucket + +``` +rclone copy /path/to/files minio:bucket +``` + +### Wasabi ### + +[Wasabi](https://wasabi.com) is a cloud-based object storage service for a +broad range of applications and use cases. Wasabi is designed for +individuals and organizations that require a high-performance, +reliable, and secure data storage infrastructure at minimal cost. + +Wasabi provides an S3 interface which can be configured for use with +rclone like this. + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +n/s> n +name> wasabi +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" +[snip] +Storage> s3 +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" +env_auth> 1 +AWS Access Key ID - leave blank for anonymous access or runtime credentials. +access_key_id> YOURACCESSKEY +AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. +secret_access_key> YOURSECRETACCESSKEY +Region to connect to. +Choose a number from below, or type in your own value + / The default endpoint - a good choice if you are unsure. + 1 | US Region, Northern Virginia or Pacific Northwest. + | Leave location constraint empty. + \ "us-east-1" +[snip] +region> us-east-1 +Endpoint for S3 API. +Leave blank if using AWS to use the default endpoint for the region. +Specify if using an S3 clone such as Ceph. +endpoint> s3.wasabisys.com +Location constraint - must be set to match the Region. Used when creating buckets only. +Choose a number from below, or type in your own value + 1 / Empty for US Region, Northern Virginia or Pacific Northwest. + \ "" +[snip] +location_constraint> +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" +[snip] +acl> +The server-side encryption algorithm used when storing this object in S3. +Choose a number from below, or type in your own value + 1 / None + \ "" + 2 / AES256 + \ "AES256" +server_side_encryption> +The storage class to use when storing objects in S3. +Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Standard storage class + \ "STANDARD" + 3 / Reduced redundancy storage class + \ "REDUCED_REDUNDANCY" + 4 / Standard Infrequent Access storage class + \ "STANDARD_IA" +storage_class> +Remote config +-------------------- +[wasabi] +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +region = us-east-1 +endpoint = s3.wasabisys.com +location_constraint = +acl = +server_side_encryption = +storage_class = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This will leave the config file looking like this. + +``` +[wasabi] +type = s3 +provider = Wasabi +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +region = +endpoint = s3.wasabisys.com +location_constraint = +acl = +server_side_encryption = +storage_class = +``` + +### Aliyun OSS / Netease NOS ### + +This describes how to set up Aliyun OSS - Netease NOS is the same +except for different endpoints. + +Note this is a pretty standard S3 setup, except for the setting of +`force_path_style = false` in the advanced config. + +``` +# rclone config +e/n/d/r/c/s/q> n +name> oss +Type of storage to configure. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio) + \ "s3" +Storage> s3 +Choose your S3 provider. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 8 / Any other S3 compatible provider + \ "Other" +provider> other +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Enter a boolean value (true or false). Press Enter for the default ("false"). +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" +env_auth> 1 +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a string value. Press Enter for the default (""). +access_key_id> xxxxxxxxxxxx +AWS Secret Access Key (password) +Leave blank for anonymous access or runtime credentials. +Enter a string value. Press Enter for the default (""). +secret_access_key> xxxxxxxxxxxxxxxxx +Region to connect to. +Leave blank if you are using an S3 clone and you don't have a region. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 1 / Use this if unsure. Will use v4 signatures and an empty region. + \ "" + 2 / Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH. + \ "other-v2-signature" +region> 1 +Endpoint for S3 API. +Required when using an S3 clone. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value +endpoint> oss-cn-shenzhen.aliyuncs.com +Location constraint - must be set to match the Region. +Leave blank if not sure. Used when creating buckets only. +Enter a string value. Press Enter for the default (""). +location_constraint> +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" +acl> 1 +Edit advanced config? (y/n) +y) Yes +n) No +y/n> y +Chunk size to use for uploading +Enter a size with suffix k,M,G,T. Press Enter for the default ("5M"). +chunk_size> +Don't store MD5 checksum with object metadata +Enter a boolean value (true or false). Press Enter for the default ("false"). +disable_checksum> +An AWS session token +Enter a string value. Press Enter for the default (""). +session_token> +Concurrency for multipart uploads. +Enter a signed integer. Press Enter for the default ("2"). +upload_concurrency> +If true use path style access if false use virtual hosted style. +Some providers (eg Aliyun OSS or Netease COS) require this. +Enter a boolean value (true or false). Press Enter for the default ("true"). +force_path_style> false +Remote config +-------------------- +[oss] +type = s3 +provider = Other +env_auth = false +access_key_id = xxxxxxxxx +secret_access_key = xxxxxxxxxxxxx +endpoint = oss-cn-shenzhen.aliyuncs.com +acl = private +force_path_style = false +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Backblaze B2 +---------------------------------------- + +B2 is [Backblaze's cloud storage system](https://www.backblaze.com/b2/). + +Paths are specified as `remote:bucket` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, eg `remote:bucket/path/to/dir`. + +Here is an example of making a b2 configuration. First run + + rclone config + +This will guide you through an interactive setup process. You will +need your account number (a short hex number) and key (a long hex +number) which you can get from the b2 control panel. + +``` +No remotes found - make a new one +n) New remote +q) Quit config +n/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" +10 / Microsoft OneDrive + \ "onedrive" +11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +12 / SSH/SFTP Connection + \ "sftp" +13 / Yandex Disk + \ "yandex" +Storage> 3 +Account ID or Application Key ID +account> 123456789abc +Application Key +key> 0123456789abcdef0123456789abcdef0123456789 +Endpoint for the service - leave blank normally. +endpoint> +Remote config +-------------------- +[remote] +account = 123456789abc +key = 0123456789abcdef0123456789abcdef0123456789 +endpoint = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This remote is called `remote` and can now be used like this + +See all buckets + + rclone lsd remote: + +Create a new bucket + + rclone mkdir remote:bucket + +List the contents of a bucket + + rclone ls remote:bucket + +Sync `/home/local/directory` to the remote bucket, deleting any +excess files in the bucket. + + rclone sync /home/local/directory remote:bucket + +### Application Keys ### + +B2 supports multiple [Application Keys for different access permission +to B2 Buckets](https://www.backblaze.com/b2/docs/application_keys.html). + +You can use these with rclone too. + +Follow Backblaze's docs to create an Application Key with the required +permission and add the `Application Key ID` as the `account` and the +`Application Key` itself as the `key`. + +Note that you must put the Application Key ID as the `account` - you +can't use the master Account ID. If you try then B2 will return 401 +errors. + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +### Modified time ### + +The modified time is stored as metadata on the object as +`X-Bz-Info-src_last_modified_millis` as milliseconds since 1970-01-01 +in the Backblaze standard. Other tools should be able to use this as +a modified time. + +Modified times are used in syncing and are fully supported except in +the case of updating a modification time on an existing object. In +this case the object will be uploaded again as B2 doesn't have an API +method to set the modification time independent of doing an upload. + +### SHA1 checksums ### + +The SHA1 checksums of the files are checked on upload and download and +will be used in the syncing process. + +Large files (bigger than the limit in `--b2-upload-cutoff`) which are +uploaded in chunks will store their SHA1 on the object as +`X-Bz-Info-large_file_sha1` as recommended by Backblaze. + +For a large file to be uploaded with an SHA1 checksum, the source +needs to support SHA1 checksums. The local disk supports SHA1 +checksums so large file transfers from local disk will have an SHA1. +See [the overview](/overview/#features) for exactly which remotes +support SHA1. + +Sources which don't support SHA1, in particular `crypt` will upload +large files without SHA1 checksums. This may be fixed in the future +(see [#1767](https://github.com/ncw/rclone/issues/1767)). + +Files sizes below `--b2-upload-cutoff` will always have an SHA1 +regardless of the source. + +### Transfers ### + +Backblaze recommends that you do lots of transfers simultaneously for +maximum speed. In tests from my SSD equipped laptop the optimum +setting is about `--transfers 32` though higher numbers may be used +for a slight speed improvement. The optimum number for you may vary +depending on your hardware, how big the files are, how much you want +to load your computer, etc. The default of `--transfers 4` is +definitely too low for Backblaze B2 though. + +Note that uploading big files (bigger than 200 MB by default) will use +a 96 MB RAM buffer by default. There can be at most `--transfers` of +these in use at any moment, so this sets the upper limit on the memory +used. + +### Versions ### + +When rclone uploads a new version of a file it creates a [new version +of it](https://www.backblaze.com/b2/docs/file_versions.html). +Likewise when you delete a file, the old version will be marked hidden +and still be available. Conversely, you may opt in to a "hard delete" +of files with the `--b2-hard-delete` flag which would permanently remove +the file instead of hiding it. + +Old versions of files, where available, are visible using the +`--b2-versions` flag. + +If you wish to remove all the old versions then you can use the +`rclone cleanup remote:bucket` command which will delete all the old +versions of files, leaving the current ones intact. You can also +supply a path and only old versions under that path will be deleted, +eg `rclone cleanup remote:bucket/path/to/stuff`. + +When you `purge` a bucket, the current and the old versions will be +deleted then the bucket will be deleted. + +However `delete` will cause the current versions of the files to +become hidden old versions. + +Here is a session showing the listing and retrieval of an old +version followed by a `cleanup` of the old versions. + +Show current version and all the versions with `--b2-versions` flag. + +``` +$ rclone -q ls b2:cleanup-test + 9 one.txt + +$ rclone -q --b2-versions ls b2:cleanup-test + 9 one.txt + 8 one-v2016-07-04-141032-000.txt + 16 one-v2016-07-04-141003-000.txt + 15 one-v2016-07-02-155621-000.txt +``` + +Retrieve an old version + +``` +$ rclone -q --b2-versions copy b2:cleanup-test/one-v2016-07-04-141003-000.txt /tmp + +$ ls -l /tmp/one-v2016-07-04-141003-000.txt +-rw-rw-r-- 1 ncw ncw 16 Jul 2 17:46 /tmp/one-v2016-07-04-141003-000.txt +``` + +Clean up all the old versions and show that they've gone. + +``` +$ rclone -q cleanup b2:cleanup-test + +$ rclone -q ls b2:cleanup-test + 9 one.txt + +$ rclone -q --b2-versions ls b2:cleanup-test + 9 one.txt +``` + +### Data usage ### + +It is useful to know how many requests are sent to the server in different scenarios. + +All copy commands send the following 4 requests: + +``` +/b2api/v1/b2_authorize_account +/b2api/v1/b2_create_bucket +/b2api/v1/b2_list_buckets +/b2api/v1/b2_list_file_names +``` + +The `b2_list_file_names` request will be sent once for every 1k files +in the remote path, providing the checksum and modification time of +the listed files. As of version 1.33 issue +[#818](https://github.com/ncw/rclone/issues/818) causes extra requests +to be sent when using B2 with Crypt. When a copy operation does not +require any files to be uploaded, no more requests will be sent. + +Uploading files that do not require chunking, will send 2 requests per +file upload: + +``` +/b2api/v1/b2_get_upload_url +/b2api/v1/b2_upload_file/ +``` + +Uploading files requiring chunking, will send 2 requests (one each to +start and finish the upload) and another 2 requests for each chunk: + +``` +/b2api/v1/b2_start_large_file +/b2api/v1/b2_get_upload_part_url +/b2api/v1/b2_upload_part/ +/b2api/v1/b2_finish_large_file +``` + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --b2-chunk-size valuee=SIZE #### + +When uploading large files chunk the file into this size. Note that +these chunks are buffered in memory and there might a maximum of +`--transfers` chunks in progress at once. 5,000,000 Bytes is the +minimim size (default 96M). + +#### --b2-upload-cutoff=SIZE #### + +Cutoff for switching to chunked upload (default 190.735 MiB == 200 +MB). Files above this size will be uploaded in chunks of +`--b2-chunk-size`. + +This value should be set no larger than 4.657GiB (== 5GB) as this is +the largest file size that can be uploaded. + + +#### --b2-test-mode=FLAG #### + +This is for debugging purposes only. + +Setting FLAG to one of the strings below will cause b2 to return +specific errors for debugging purposes. + + * `fail_some_uploads` + * `expire_some_account_authorization_tokens` + * `force_cap_exceeded` + +These will be set in the `X-Bz-Test-Mode` header which is documented +in the [b2 integrations +checklist](https://www.backblaze.com/b2/docs/integration_checklist.html). + +#### --b2-versions #### + +When set rclone will show and act on older versions of files. For example + +Listing without `--b2-versions` + +``` +$ rclone -q ls b2:cleanup-test + 9 one.txt +``` + +And with + +``` +$ rclone -q --b2-versions ls b2:cleanup-test + 9 one.txt + 8 one-v2016-07-04-141032-000.txt + 16 one-v2016-07-04-141003-000.txt + 15 one-v2016-07-02-155621-000.txt +``` + +Showing that the current version is unchanged but older versions can +be seen. These have the UTC date that they were uploaded to the +server to the nearest millisecond appended to them. + +Note that when using `--b2-versions` no file write operations are +permitted, so you can't upload files or delete them. + +Box +----------------------------------------- + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +The initial setup for Box involves getting a token from Box which you +need to do in your browser. `rclone config` walks you through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Box + \ "box" + 5 / Dropbox + \ "dropbox" + 6 / Encrypt/Decrypt a remote + \ "crypt" + 7 / FTP Connection + \ "ftp" + 8 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 9 / Google Drive + \ "drive" +10 / Hubic + \ "hubic" +11 / Local Disk + \ "local" +12 / Microsoft OneDrive + \ "onedrive" +13 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +14 / SSH/SFTP Connection + \ "sftp" +15 / Yandex Disk + \ "yandex" +16 / http Connection + \ "http" +Storage> box +Box App Client Id - leave blank normally. +client_id> +Box App Client Secret - leave blank normally. +client_secret> +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +client_id = +client_secret = +token = {"access_token":"XXX","token_type":"bearer","refresh_token":"XXX","expiry":"XXX"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Box. This only runs from the moment it opens +your browser to the moment you get back the verification code. This +is on `http://127.0.0.1:53682/` and this it may require you to unblock +it temporarily if you are running a host firewall. + +Once configured you can then use `rclone` like this, + +List directories in top level of your Box + + rclone lsd remote: + +List all the files in your Box + + rclone ls remote: + +To copy a local directory to an Box directory called backup + + rclone copy /home/source remote:backup + +### Invalid refresh token ### + +According to the [box docs](https://developer.box.com/v2.0/docs/oauth-20#section-6-using-the-access-and-refresh-tokens): + +> Each refresh_token is valid for one use in 60 days. + +This means that if you + + * Don't use the box remote for 60 days + * Copy the config file with a box refresh token in and use it in two places + * Get an error on a token refresh + +then rclone will return an error which includes the text `Invalid +refresh token`. + +To fix this you will need to use oauth2 again to update the refresh +token. You can use the methods in [the remote setup +docs](https://rclone.org/remote_setup/), bearing in mind that if you use the copy the +config file method, you should not use that remote on the computer you +did the authentication on. + +Here is how to do it. + +``` +$ rclone config +Current remotes: + +Name Type +==== ==== +remote box + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> e +Choose a number from below, or type in an existing value + 1 > remote +remote> remote +-------------------- +[remote] +type = box +token = {"access_token":"XXX","token_type":"bearer","refresh_token":"XXX","expiry":"2017-07-08T23:40:08.059167677+01:00"} +-------------------- +Edit remote +Value "client_id" = "" +Edit? (y/n)> +y) Yes +n) No +y/n> n +Value "client_secret" = "" +Edit? (y/n)> +y) Yes +n) No +y/n> n +Remote config +Already have a token - refresh? +y) Yes +n) No +y/n> y +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +type = box +token = {"access_token":"YYY","token_type":"bearer","refresh_token":"YYY","expiry":"2017-07-23T12:22:29.259137901+01:00"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +### Modified time and hashes ### + +Box allows modification times to be set on objects accurate to 1 +second. These will be used to detect whether objects need syncing or +not. + +Box supports SHA1 type hashes, so you can use the `--checksum` +flag. + +### Transfers ### + +For files above 50MB rclone will use a chunked transfer. Rclone will +upload up to `--transfers` chunks at the same time (shared among all +the multipart uploads). Chunks are buffered in memory and are +normally 8MB so increasing `--transfers` will increase memory use. + +### Deleting files ### + +Depending on the enterprise settings for your user, the item will +either be actually deleted from Box or moved to the trash. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --box-upload-cutoff=SIZE #### + +Cutoff for switching to chunked upload - must be >= 50MB. The default +is 50MB. + +#### --box-commit-retries int #### + +Max number of times to try committing a multipart file. (default 100) + +### Limitations ### + +Note that Box is case insensitive so you can't have a file called +"Hello.doc" and one called "hello.doc". + +Box file names can't have the `\` character in. rclone maps this to +and from an identical looking unicode equivalent `＼`. + +Box only supports filenames up to 255 characters in length. + +Cache (BETA) +----------------------------------------- + +The `cache` remote wraps another existing remote and stores file structure +and its data for long running tasks like `rclone mount`. + +To get started you just need to have an existing remote which can be configured +with `cache`. + +Here is an example of how to make a remote called `test-cache`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +n/r/c/s/q> n +name> test-cache +Type of storage to configure. +Choose a number from below, or type in your own value +... + 5 / Cache a remote + \ "cache" +... +Storage> 5 +Remote to cache. +Normally should contain a ':' and a path, eg "myremote:path/to/dir", +"myremote:bucket" or maybe "myremote:" (not recommended). +remote> local:/test +Optional: The URL of the Plex server +plex_url> http://127.0.0.1:32400 +Optional: The username of the Plex user +plex_username> dummyusername +Optional: The password of the Plex user +y) Yes type in my own password +g) Generate random password +n) No leave this optional password blank +y/g/n> y +Enter the password: +password: +Confirm the password: +password: +The size of a chunk. Lower value good for slow connections but can affect seamless reading. +Default: 5M +Choose a number from below, or type in your own value + 1 / 1MB + \ "1m" + 2 / 5 MB + \ "5M" + 3 / 10 MB + \ "10M" +chunk_size> 2 +How much time should object info (file size, file hashes etc) be stored in cache. Use a very high value if you don't plan on changing the source FS from outside the cache. +Accepted units are: "s", "m", "h". +Default: 5m +Choose a number from below, or type in your own value + 1 / 1 hour + \ "1h" + 2 / 24 hours + \ "24h" + 3 / 24 hours + \ "48h" +info_age> 2 +The maximum size of stored chunks. When the storage grows beyond this size, the oldest chunks will be deleted. +Default: 10G +Choose a number from below, or type in your own value + 1 / 500 MB + \ "500M" + 2 / 1 GB + \ "1G" + 3 / 10 GB + \ "10G" +chunk_total_size> 3 +Remote config +-------------------- +[test-cache] +remote = local:/test +plex_url = http://127.0.0.1:32400 +plex_username = dummyusername +plex_password = *** ENCRYPTED *** +chunk_size = 5M +info_age = 48h +chunk_total_size = 10G +``` + +You can then use it like this, + +List directories in top level of your drive + + rclone lsd test-cache: + +List all the files in your drive + + rclone ls test-cache: + +To start a cached mount + + rclone mount --allow-other test-cache: /var/tmp/test-cache + +### Write Features ### + +### Offline uploading ### + +In an effort to make writing through cache more reliable, the backend +now supports this feature which can be activated by specifying a +`cache-tmp-upload-path`. + +A files goes through these states when using this feature: + +1. An upload is started (usually by copying a file on the cache remote) +2. When the copy to the temporary location is complete the file is part +of the cached remote and looks and behaves like any other file (reading included) +3. After `cache-tmp-wait-time` passes and the file is next in line, `rclone move` +is used to move the file to the cloud provider +4. Reading the file still works during the upload but most modifications on it will be prohibited +5. Once the move is complete the file is unlocked for modifications as it +becomes as any other regular file +6. If the file is being read through `cache` when it's actually +deleted from the temporary path then `cache` will simply swap the source +to the cloud provider without interrupting the reading (small blip can happen though) + +Files are uploaded in sequence and only one file is uploaded at a time. +Uploads will be stored in a queue and be processed based on the order they were added. +The queue and the temporary storage is persistent across restarts and even purges of the cache. + +### Write Support ### + +Writes are supported through `cache`. +One caveat is that a mounted cache remote does not add any retry or fallback +mechanism to the upload operation. This will depend on the implementation +of the wrapped remote. Consider using `Offline uploading` for reliable writes. + +One special case is covered with `cache-writes` which will cache the file +data at the same time as the upload when it is enabled making it available +from the cache store immediately once the upload is finished. + +### Read Features ### + +#### Multiple connections #### + +To counter the high latency between a local PC where rclone is running +and cloud providers, the cache remote can split multiple requests to the +cloud provider for smaller file chunks and combines them together locally +where they can be available almost immediately before the reader usually +needs them. + +This is similar to buffering when media files are played online. Rclone +will stay around the current marker but always try its best to stay ahead +and prepare the data before. + +#### Plex Integration #### + +There is a direct integration with Plex which allows cache to detect during reading +if the file is in playback or not. This helps cache to adapt how it queries +the cloud provider depending on what is needed for. + +Scans will have a minimum amount of workers (1) while in a confirmed playback cache +will deploy the configured number of workers. + +This integration opens the doorway to additional performance improvements +which will be explored in the near future. + +**Note:** If Plex options are not configured, `cache` will function with its +configured options without adapting any of its settings. + +How to enable? Run `rclone config` and add all the Plex options (endpoint, username +and password) in your remote and it will be automatically enabled. + +Affected settings: +- `cache-workers`: _Configured value_ during confirmed playback or _1_ all the other times + +### Known issues ### + +#### Mount and --dir-cache-time #### + +--dir-cache-time controls the first layer of directory caching which works at the mount layer. +Being an independent caching mechanism from the `cache` backend, it will manage its own entries +based on the configured time. + +To avoid getting in a scenario where dir cache has obsolete data and cache would have the correct +one, try to set `--dir-cache-time` to a lower time than `--cache-info-age`. Default values are +already configured in this way. + +#### Windows support - Experimental #### + +There are a couple of issues with Windows `mount` functionality that still require some investigations. +It should be considered as experimental thus far as fixes come in for this OS. + +Most of the issues seem to be related to the difference between filesystems +on Linux flavors and Windows as cache is heavily dependant on them. + +Any reports or feedback on how cache behaves on this OS is greatly appreciated. + +- https://github.com/ncw/rclone/issues/1935 +- https://github.com/ncw/rclone/issues/1907 +- https://github.com/ncw/rclone/issues/1834 + +#### Risk of throttling #### + +Future iterations of the cache backend will make use of the pooling functionality +of the cloud provider to synchronize and at the same time make writing through it +more tolerant to failures. + +There are a couple of enhancements in track to add these but in the meantime +there is a valid concern that the expiring cache listings can lead to cloud provider +throttles or bans due to repeated queries on it for very large mounts. + +Some recommendations: +- don't use a very small interval for entry informations (`--cache-info-age`) +- while writes aren't yet optimised, you can still write through `cache` which gives you the advantage +of adding the file in the cache at the same time if configured to do so. + +Future enhancements: + +- https://github.com/ncw/rclone/issues/1937 +- https://github.com/ncw/rclone/issues/1936 + +#### cache and crypt #### + +One common scenario is to keep your data encrypted in the cloud provider +using the `crypt` remote. `crypt` uses a similar technique to wrap around +an existing remote and handles this translation in a seamless way. + +There is an issue with wrapping the remotes in this order: +**cloud remote** -> **crypt** -> **cache** + +During testing, I experienced a lot of bans with the remotes in this order. +I suspect it might be related to how crypt opens files on the cloud provider +which makes it think we're downloading the full file instead of small chunks. +Organizing the remotes in this order yelds better results: +**cloud remote** -> **cache** -> **crypt** + +### Cache and Remote Control (--rc) ### +Cache supports the new `--rc` mode in rclone and can be remote controlled through the following end points: +By default, the listener is disabled if you do not add the flag. + +### rc cache/expire +Purge a remote from the cache backend. Supports either a directory or a file. +It supports both encrypted and unencrypted file names if cache is wrapped by crypt. + +Params: + - **remote** = path to remote **(required)** + - **withData** = true/false to delete cached data (chunks) as well _(optional, false by default)_ + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --cache-db-path=PATH #### + +Path to where the file structure metadata (DB) is stored locally. The remote +name is used as the DB file name. + +**Default**: /cache-backend/ +**Example**: /.cache/cache-backend/test-cache + +#### --cache-chunk-path=PATH #### + +Path to where partial file data (chunks) is stored locally. The remote +name is appended to the final path. + +This config follows the `--cache-db-path`. If you specify a custom +location for `--cache-db-path` and don't specify one for `--cache-chunk-path` +then `--cache-chunk-path` will use the same path as `--cache-db-path`. + +**Default**: /cache-backend/ +**Example**: /.cache/cache-backend/test-cache + +#### --cache-db-purge #### + +Flag to clear all the cached data for this remote before. + +**Default**: not set + +#### --cache-chunk-size=SIZE #### + +The size of a chunk (partial file data). Use lower numbers for slower +connections. If the chunk size is changed, any downloaded chunks will be invalid and cache-chunk-path will need to be cleared or unexpected EOF errors will occur. + +**Default**: 5M + +#### --cache-total-chunk-size=SIZE #### + +The total size that the chunks can take up on the local disk. If `cache` +exceeds this value then it will start to the delete the oldest chunks until +it goes under this value. + +**Default**: 10G + +#### --cache-chunk-clean-interval=DURATION #### + +How often should `cache` perform cleanups of the chunk storage. The default value +should be ok for most people. If you find that `cache` goes over `cache-total-chunk-size` +too often then try to lower this value to force it to perform cleanups more often. + +**Default**: 1m + +#### --cache-info-age=DURATION #### + +How long to keep file structure information (directory listings, file size, +mod times etc) locally. + +If all write operations are done through `cache` then you can safely make +this value very large as the cache store will also be updated in real time. + +**Default**: 6h + +#### --cache-read-retries=RETRIES #### + +How many times to retry a read from a cache storage. + +Since reading from a `cache` stream is independent from downloading file data, +readers can get to a point where there's no more data in the cache. +Most of the times this can indicate a connectivity issue if `cache` isn't +able to provide file data anymore. + +For really slow connections, increase this to a point where the stream is +able to provide data but your experience will be very stuttering. + +**Default**: 10 + +#### --cache-workers=WORKERS #### + +How many workers should run in parallel to download chunks. + +Higher values will mean more parallel processing (better CPU needed) and +more concurrent requests on the cloud provider. +This impacts several aspects like the cloud provider API limits, more stress +on the hardware that rclone runs on but it also means that streams will +be more fluid and data will be available much more faster to readers. + +**Note**: If the optional Plex integration is enabled then this setting +will adapt to the type of reading performed and the value specified here will be used +as a maximum number of workers to use. +**Default**: 4 + +#### --cache-chunk-no-memory #### + +By default, `cache` will keep file data during streaming in RAM as well +to provide it to readers as fast as possible. + +This transient data is evicted as soon as it is read and the number of +chunks stored doesn't exceed the number of workers. However, depending +on other settings like `cache-chunk-size` and `cache-workers` this footprint +can increase if there are parallel streams too (multiple files being read +at the same time). + +If the hardware permits it, use this feature to provide an overall better +performance during streaming but it can also be disabled if RAM is not +available on the local machine. + +**Default**: not set + +#### --cache-rps=NUMBER #### + +This setting places a hard limit on the number of requests per second that `cache` +will be doing to the cloud provider remote and try to respect that value +by setting waits between reads. + +If you find that you're getting banned or limited on the cloud provider +through cache and know that a smaller number of requests per second will +allow you to work with it then you can use this setting for that. + +A good balance of all the other settings should make this +setting useless but it is available to set for more special cases. + +**NOTE**: This will limit the number of requests during streams but other +API calls to the cloud provider like directory listings will still pass. + +**Default**: disabled + +#### --cache-writes #### + +If you need to read files immediately after you upload them through `cache` +you can enable this flag to have their data stored in the cache store at the +same time during upload. + +**Default**: not set + +#### --cache-tmp-upload-path=PATH #### + +This is the path where `cache` will use as a temporary storage for new files +that need to be uploaded to the cloud provider. + +Specifying a value will enable this feature. Without it, it is completely disabled +and files will be uploaded directly to the cloud provider + +**Default**: empty + +#### --cache-tmp-wait-time=DURATION #### + +This is the duration that a file must wait in the temporary location +_cache-tmp-upload-path_ before it is selected for upload. + +Note that only one file is uploaded at a time and it can take longer to +start the upload if a queue formed for this purpose. + +**Default**: 15m + +#### --cache-db-wait-time=DURATION #### + +Only one process can have the DB open at any one time, so rclone waits +for this duration for the DB to become available before it gives an +error. + +If you set it to 0 then it will wait forever. + +**Default**: 1s + +Crypt +---------------------------------------- + +The `crypt` remote encrypts and decrypts another remote. + +To use it first set up the underlying remote following the config +instructions for that remote. You can also use a local pathname +instead of a remote which will encrypt and decrypt from that directory +which might be useful for encrypting onto a USB stick for example. + +First check your chosen remote is working - we'll call it +`remote:path` in these docs. Note that anything inside `remote:path` +will be encrypted and anything outside won't. This means that if you +are using a bucket based remote (eg S3, B2, swift) then you should +probably put the bucket in the remote `s3:bucket`. If you just use +`s3:` then rclone will make encrypted bucket names too (if using file +name encryption) which may or may not be what you want. + +Now configure `crypt` using `rclone config`. We will call this one +`secret` to differentiate it from the `remote`. + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> secret +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" +10 / Microsoft OneDrive + \ "onedrive" +11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +12 / SSH/SFTP Connection + \ "sftp" +13 / Yandex Disk + \ "yandex" +Storage> 5 +Remote to encrypt/decrypt. +Normally should contain a ':' and a path, eg "myremote:path/to/dir", +"myremote:bucket" or maybe "myremote:" (not recommended). +remote> remote:path +How to encrypt the filenames. +Choose a number from below, or type in your own value + 1 / Don't encrypt the file names. Adds a ".bin" extension only. + \ "off" + 2 / Encrypt the filenames see the docs for the details. + \ "standard" + 3 / Very simple filename obfuscation. + \ "obfuscate" +filename_encryption> 2 +Option to either encrypt directory names or leave them intact. +Choose a number from below, or type in your own value + 1 / Encrypt directory names. + \ "true" + 2 / Don't encrypt directory names, leave them intact. + \ "false" +filename_encryption> 1 +Password or pass phrase for encryption. +y) Yes type in my own password +g) Generate random password +y/g> y +Enter the password: +password: +Confirm the password: +password: +Password or pass phrase for salt. Optional but recommended. +Should be different to the previous password. +y) Yes type in my own password +g) Generate random password +n) No leave this optional password blank +y/g/n> g +Password strength in bits. +64 is just about memorable +128 is secure +1024 is the maximum +Bits> 128 +Your password is: JAsJvRcgR-_veXNfy_sGmQ +Use this password? +y) Yes +n) No +y/n> y +Remote config +-------------------- +[secret] +remote = remote:path +filename_encryption = standard +password = *** ENCRYPTED *** +password2 = *** ENCRYPTED *** +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +**Important** The password is stored in the config file is lightly +obscured so it isn't immediately obvious what it is. It is in no way +secure unless you use config file encryption. + +A long passphrase is recommended, or you can use a random one. Note +that if you reconfigure rclone with the same passwords/passphrases +elsewhere it will be compatible - all the secrets used are derived +from those two passwords/passphrases. + +Note that rclone does not encrypt + + * file length - this can be calcuated within 16 bytes + * modification time - used for syncing + +## Specifying the remote ## + +In normal use, make sure the remote has a `:` in. If you specify the +remote without a `:` then rclone will use a local directory of that +name. So if you use a remote of `/path/to/secret/files` then rclone +will encrypt stuff to that directory. If you use a remote of `name` +then rclone will put files in a directory called `name` in the current +directory. + +If you specify the remote as `remote:path/to/dir` then rclone will +store encrypted files in `path/to/dir` on the remote. If you are using +file name encryption, then when you save files to +`secret:subdir/subfile` this will store them in the unencrypted path +`path/to/dir` but the `subdir/subpath` bit will be encrypted. + +Note that unless you want encrypted bucket names (which are difficult +to manage because you won't know what directory they represent in web +interfaces etc), you should probably specify a bucket, eg +`remote:secretbucket` when using bucket based remotes such as S3, +Swift, Hubic, B2, GCS. + +## Example ## + +To test I made a little directory of files using "standard" file name +encryption. + +``` +plaintext/ +├── file0.txt +├── file1.txt +└── subdir + ├── file2.txt + ├── file3.txt + └── subsubdir + └── file4.txt +``` + +Copy these to the remote and list them back + +``` +$ rclone -q copy plaintext secret: +$ rclone -q ls secret: + 7 file1.txt + 6 file0.txt + 8 subdir/file2.txt + 10 subdir/subsubdir/file4.txt + 9 subdir/file3.txt +``` + +Now see what that looked like when encrypted + +``` +$ rclone -q ls remote:path + 55 hagjclgavj2mbiqm6u6cnjjqcg + 54 v05749mltvv1tf4onltun46gls + 57 86vhrsv86mpbtd3a0akjuqslj8/dlj7fkq4kdq72emafg7a7s41uo + 58 86vhrsv86mpbtd3a0akjuqslj8/7uu829995du6o42n32otfhjqp4/b9pausrfansjth5ob3jkdqd4lc + 56 86vhrsv86mpbtd3a0akjuqslj8/8njh1sk437gttmep3p70g81aps +``` + +Note that this retains the directory structure which means you can do this + +``` +$ rclone -q ls secret:subdir + 8 file2.txt + 9 file3.txt + 10 subsubdir/file4.txt +``` + +If don't use file name encryption then the remote will look like this +- note the `.bin` extensions added to prevent the cloud provider +attempting to interpret the data. + +``` +$ rclone -q ls remote:path + 54 file0.txt.bin + 57 subdir/file3.txt.bin + 56 subdir/file2.txt.bin + 58 subdir/subsubdir/file4.txt.bin + 55 file1.txt.bin +``` + +### File name encryption modes ### + +Here are some of the features of the file name encryption modes + +Off + + * doesn't hide file names or directory structure + * allows for longer file names (~246 characters) + * can use sub paths and copy single files + +Standard + + * file names encrypted + * file names can't be as long (~143 characters) + * can use sub paths and copy single files + * directory structure visible + * identical files names will have identical uploaded names + * can use shortcuts to shorten the directory recursion + +Obfuscation + +This is a simple "rotate" of the filename, with each file having a rot +distance based on the filename. We store the distance at the beginning +of the filename. So a file called "hello" may become "53.jgnnq" + +This is not a strong encryption of filenames, but it may stop automated +scanning tools from picking up on filename patterns. As such it's an +intermediate between "off" and "standard". The advantage is that it +allows for longer path segment names. + +There is a possibility with some unicode based filenames that the +obfuscation is weak and may map lower case characters to upper case +equivalents. You can not rely on this for strong protection. + + * file names very lightly obfuscated + * file names can be longer than standard encryption + * can use sub paths and copy single files + * directory structure visible + * identical files names will have identical uploaded names + +Cloud storage systems have various limits on file name length and +total path length which you are more likely to hit using "Standard" +file name encryption. If you keep your file names to below 156 +characters in length then you should be OK on all providers. + +There may be an even more secure file name encryption mode in the +future which will address the long file name problem. + +### Directory name encryption ### +Crypt offers the option of encrypting dir names or leaving them intact. +There are two options: + +True + +Encrypts the whole file path including directory names +Example: +`1/12/123.txt` is encrypted to +`p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0` + +False + +Only encrypts file names, skips directory names +Example: +`1/12/123.txt` is encrypted to +`1/12/qgm4avr35m5loi1th53ato71v0` + + +### Modified time and hashes ### + +Crypt stores modification times using the underlying remote so support +depends on that. + +Hashes are not stored for crypt. However the data integrity is +protected by an extremely strong crypto authenticator. + +Note that you should use the `rclone cryptcheck` command to check the +integrity of a crypted remote instead of `rclone check` which can't +check the checksums properly. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --crypt-show-mapping #### + +If this flag is set then for each file that the remote is asked to +list, it will log (at level INFO) a line stating the decrypted file +name and the encrypted file name. + +This is so you can work out which encrypted names are which decrypted +names just in case you need to do something with the encrypted file +names, or for debugging purposes. + +## Backing up a crypted remote ## + +If you wish to backup a crypted remote, it it recommended that you use +`rclone sync` on the encrypted files, and make sure the passwords are +the same in the new encrypted remote. + +This will have the following advantages + + * `rclone sync` will check the checksums while copying + * you can use `rclone check` between the encrypted remotes + * you don't decrypt and encrypt unnecessarily + +For example, let's say you have your original remote at `remote:` with +the encrypted version at `eremote:` with path `remote:crypt`. You +would then set up the new remote `remote2:` and then the encrypted +version `eremote2:` with path `remote2:crypt` using the same passwords +as `eremote:`. + +To sync the two remotes you would do + + rclone sync remote:crypt remote2:crypt + +And to check the integrity you would do + + rclone check remote:crypt remote2:crypt + +## File formats ## + +### File encryption ### + +Files are encrypted 1:1 source file to destination object. The file +has a header and is divided into chunks. + +#### Header #### + + * 8 bytes magic string `RCLONE\x00\x00` + * 24 bytes Nonce (IV) + +The initial nonce is generated from the operating systems crypto +strong random number generator. The nonce is incremented for each +chunk read making sure each nonce is unique for each block written. +The chance of a nonce being re-used is minuscule. If you wrote an +exabyte of data (10¹⁸ bytes) you would have a probability of +approximately 2×10⁻³² of re-using a nonce. + +#### Chunk #### + +Each chunk will contain 64kB of data, except for the last one which +may have less data. The data chunk is in standard NACL secretbox +format. Secretbox uses XSalsa20 and Poly1305 to encrypt and +authenticate messages. + +Each chunk contains: + + * 16 Bytes of Poly1305 authenticator + * 1 - 65536 bytes XSalsa20 encrypted data + +64k chunk size was chosen as the best performing chunk size (the +authenticator takes too much time below this and the performance drops +off due to cache effects above this). Note that these chunks are +buffered in memory so they can't be too big. + +This uses a 32 byte (256 bit key) key derived from the user password. + +#### Examples #### + +1 byte file will encrypt to + + * 32 bytes header + * 17 bytes data chunk + +49 bytes total + +1MB (1048576 bytes) file will encrypt to + + * 32 bytes header + * 16 chunks of 65568 bytes + +1049120 bytes total (a 0.05% overhead). This is the overhead for big +files. + +### Name encryption ### + +File names are encrypted segment by segment - the path is broken up +into `/` separated strings and these are encrypted individually. + +File segments are padded using using PKCS#7 to a multiple of 16 bytes +before encryption. + +They are then encrypted with EME using AES with 256 bit key. EME +(ECB-Mix-ECB) is a wide-block encryption mode presented in the 2003 +paper "A Parallelizable Enciphering Mode" by Halevi and Rogaway. + +This makes for deterministic encryption which is what we want - the +same filename must encrypt to the same thing otherwise we can't find +it on the cloud storage system. + +This means that + + * filenames with the same name will encrypt the same + * filenames which start the same won't have a common prefix + +This uses a 32 byte key (256 bits) and a 16 byte (128 bits) IV both of +which are derived from the user password. + +After encryption they are written out using a modified version of +standard `base32` encoding as described in RFC4648. The standard +encoding is modified in two ways: + + * it becomes lower case (no-one likes upper case filenames!) + * we strip the padding character `=` + +`base32` is used rather than the more efficient `base64` so rclone can be +used on case insensitive remotes (eg Windows, Amazon Drive). + +### Key derivation ### + +Rclone uses `scrypt` with parameters `N=16384, r=8, p=1` with an +optional user supplied salt (password2) to derive the 32+32+16 = 80 +bytes of key material required. If the user doesn't supply a salt +then rclone uses an internal one. + +`scrypt` makes it impractical to mount a dictionary attack on rclone +encrypted data. For full protection against this you should always use +a salt. + +Dropbox +--------------------------------- + +Paths are specified as `remote:path` + +Dropbox paths may be as deep as required, eg +`remote:directory/subdirectory`. + +The initial setup for dropbox involves getting a token from Dropbox +which you need to do in your browser. `rclone config` walks you +through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +n) New remote +d) Delete remote +q) Quit config +e/n/d/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" +10 / Microsoft OneDrive + \ "onedrive" +11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +12 / SSH/SFTP Connection + \ "sftp" +13 / Yandex Disk + \ "yandex" +Storage> 4 +Dropbox App Key - leave blank normally. +app_key> +Dropbox App Secret - leave blank normally. +app_secret> +Remote config +Please visit: +https://www.dropbox.com/1/oauth2/authorize?client_id=XXXXXXXXXXXXXXX&response_type=code +Enter the code: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXXXXXXXX +-------------------- +[remote] +app_key = +app_secret = +token = XXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXX_XXXXXXXXXXXXXXXXXXXXXXXXXXXXX +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +You can then use it like this, + +List directories in top level of your dropbox + + rclone lsd remote: + +List all the files in your dropbox + + rclone ls remote: + +To copy a local directory to a dropbox directory called backup + + rclone copy /home/source remote:backup + +### Dropbox for business ### + +Rclone supports Dropbox for business and Team Folders. + +When using Dropbox for business `remote:` and `remote:path/to/file` +will refer to your personal folder. + +If you wish to see Team Folders you must use a leading `/` in the +path, so `rclone lsd remote:/` will refer to the root and show you all +Team Folders and your User Folder. + +You can then use team folders like this `remote:/TeamFolder` and +`remote:/TeamFolder/path/to/file`. + +A leading `/` for a Dropbox personal account will do nothing, but it +will take an extra HTTP transaction so it should be avoided. + +### Modified time and Hashes ### + +Dropbox supports modified times, but the only way to set a +modification time is to re-upload the file. + +This means that if you uploaded your data with an older version of +rclone which didn't support the v2 API and modified times, rclone will +decide to upload all your old data to fix the modification times. If +you don't want this to happen use `--size-only` or `--checksum` flag +to stop it. + +Dropbox supports [its own hash +type](https://www.dropbox.com/developers/reference/content-hash) which +is checked for all transfers. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --dropbox-chunk-size=SIZE #### + +Any files larger than this will be uploaded in chunks of this +size. The default is 48MB. The maximum is 150MB. + +Note that chunks are buffered in memory (one at a time) so rclone can +deal with retries. Setting this larger will increase the speed +slightly (at most 10% for 128MB in tests) at the cost of using more +memory. It can be set smaller if you are tight on memory. + +### Limitations ### + +Note that Dropbox is case insensitive so you can't have a file called +"Hello.doc" and one called "hello.doc". + +There are some file names such as `thumbs.db` which Dropbox can't +store. There is a full list of them in the ["Ignored Files" section +of this document](https://www.dropbox.com/en/help/145). Rclone will +issue an error message `File name disallowed - not uploading` if it +attempts to upload one of those file names, but the sync won't fail. + +If you have more than 10,000 files in a directory then `rclone purge +dropbox:dir` will return the error `Failed to purge: There are too +many files involved in this operation`. As a work-around do an +`rclone delete dropbox:dir` followed by an `rclone rmdir dropbox:dir`. + +FTP +------------------------------ + +FTP is the File Transfer Protocol. FTP support is provided using the +[github.com/jlaffaye/ftp](https://godoc.org/github.com/jlaffaye/ftp) +package. + +Here is an example of making an FTP configuration. First run + + rclone config + +This will guide you through an interactive setup process. An FTP remote only +needs a host together with and a username and a password. With anonymous FTP +server, you will need to use `anonymous` as username and your email address as +the password. + +``` +No remotes found - make a new one +n) New remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +n/r/c/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / FTP Connection + \ "ftp" + 7 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 8 / Google Drive + \ "drive" + 9 / Hubic + \ "hubic" +10 / Local Disk + \ "local" +11 / Microsoft OneDrive + \ "onedrive" +12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +13 / SSH/SFTP Connection + \ "sftp" +14 / Yandex Disk + \ "yandex" +Storage> ftp +FTP host to connect to +Choose a number from below, or type in your own value + 1 / Connect to ftp.example.com + \ "ftp.example.com" +host> ftp.example.com +FTP username, leave blank for current username, ncw +user> +FTP port, leave blank to use default (21) +port> +FTP password +y) Yes type in my own password +g) Generate random password +y/g> y +Enter the password: +password: +Confirm the password: +password: +Remote config +-------------------- +[remote] +host = ftp.example.com +user = +port = +pass = *** ENCRYPTED *** +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This remote is called `remote` and can now be used like this + +See all directories in the home directory + + rclone lsd remote: + +Make a new directory + + rclone mkdir remote:path/to/directory + +List the contents of a directory + + rclone ls remote:path/to/directory + +Sync `/home/local/directory` to the remote directory, deleting any +excess files in the directory. + + rclone sync /home/local/directory remote:directory + +### Modified time ### + +FTP does not support modified times. Any times you see on the server +will be time of upload. + +### Checksums ### + +FTP does not support any checksums. + +### Limitations ### + +Note that since FTP isn't HTTP based the following flags don't work +with it: `--dump-headers`, `--dump-bodies`, `--dump-auth` + +Note that `--timeout` isn't supported (but `--contimeout` is). + +Note that `--bind` isn't supported. + +FTP could support server side move but doesn't yet. + +Google Cloud Storage +------------------------------------------------- + +Paths are specified as `remote:bucket` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, eg `remote:bucket/path/to/dir`. + +The initial setup for google cloud storage involves getting a token from Google Cloud Storage +which you need to do in your browser. `rclone config` walks you +through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +n) New remote +d) Delete remote +q) Quit config +e/n/d/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" +10 / Microsoft OneDrive + \ "onedrive" +11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +12 / SSH/SFTP Connection + \ "sftp" +13 / Yandex Disk + \ "yandex" +Storage> 6 +Google Application Client Id - leave blank normally. +client_id> +Google Application Client Secret - leave blank normally. +client_secret> +Project number optional - needed only for list/create/delete buckets - see your developer console. +project_number> 12345678 +Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. +service_account_file> +Access Control List for new objects. +Choose a number from below, or type in your own value + 1 / Object owner gets OWNER access, and all Authenticated Users get READER access. + \ "authenticatedRead" + 2 / Object owner gets OWNER access, and project team owners get OWNER access. + \ "bucketOwnerFullControl" + 3 / Object owner gets OWNER access, and project team owners get READER access. + \ "bucketOwnerRead" + 4 / Object owner gets OWNER access [default if left blank]. + \ "private" + 5 / Object owner gets OWNER access, and project team members get access according to their roles. + \ "projectPrivate" + 6 / Object owner gets OWNER access, and all Users get READER access. + \ "publicRead" +object_acl> 4 +Access Control List for new buckets. +Choose a number from below, or type in your own value + 1 / Project team owners get OWNER access, and all Authenticated Users get READER access. + \ "authenticatedRead" + 2 / Project team owners get OWNER access [default if left blank]. + \ "private" + 3 / Project team members get access according to their roles. + \ "projectPrivate" + 4 / Project team owners get OWNER access, and all Users get READER access. + \ "publicRead" + 5 / Project team owners get OWNER access, and all Users get WRITER access. + \ "publicReadWrite" +bucket_acl> 2 +Location for the newly created buckets. +Choose a number from below, or type in your own value + 1 / Empty for default location (US). + \ "" + 2 / Multi-regional location for Asia. + \ "asia" + 3 / Multi-regional location for Europe. + \ "eu" + 4 / Multi-regional location for United States. + \ "us" + 5 / Taiwan. + \ "asia-east1" + 6 / Tokyo. + \ "asia-northeast1" + 7 / Singapore. + \ "asia-southeast1" + 8 / Sydney. + \ "australia-southeast1" + 9 / Belgium. + \ "europe-west1" +10 / London. + \ "europe-west2" +11 / Iowa. + \ "us-central1" +12 / South Carolina. + \ "us-east1" +13 / Northern Virginia. + \ "us-east4" +14 / Oregon. + \ "us-west1" +location> 12 +The storage class to use when storing objects in Google Cloud Storage. +Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Multi-regional storage class + \ "MULTI_REGIONAL" + 3 / Regional storage class + \ "REGIONAL" + 4 / Nearline storage class + \ "NEARLINE" + 5 / Coldline storage class + \ "COLDLINE" + 6 / Durable reduced availability storage class + \ "DURABLE_REDUCED_AVAILABILITY" +storage_class> 5 +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine or Y didn't work +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +type = google cloud storage +client_id = +client_secret = +token = {"AccessToken":"xxxx.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","RefreshToken":"x/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_xxxxxxxxx","Expiry":"2014-07-17T20:49:14.929208288+01:00","Extra":null} +project_number = 12345678 +object_acl = private +bucket_acl = private +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Google if you use auto config mode. This only +runs from the moment it opens your browser to the moment you get back +the verification code. This is on `http://127.0.0.1:53682/` and this +it may require you to unblock it temporarily if you are running a host +firewall, or use manual mode. + +This remote is called `remote` and can now be used like this + +See all the buckets in your project + + rclone lsd remote: + +Make a new bucket + + rclone mkdir remote:bucket + +List the contents of a bucket + + rclone ls remote:bucket + +Sync `/home/local/directory` to the remote bucket, deleting any excess +files in the bucket. + + rclone sync /home/local/directory remote:bucket + +### Service Account support ### + +You can set up rclone with Google Cloud Storage in an unattended mode, +i.e. not tied to a specific end-user Google account. This is useful +when you want to synchronise files onto machines that don't have +actively logged-in users, for example build machines. + +To get credentials for Google Cloud Platform +[IAM Service Accounts](https://cloud.google.com/iam/docs/service-accounts), +please head to the +[Service Account](https://console.cloud.google.com/permissions/serviceaccounts) +section of the Google Developer Console. Service Accounts behave just +like normal `User` permissions in +[Google Cloud Storage ACLs](https://cloud.google.com/storage/docs/access-control), +so you can limit their access (e.g. make them read only). After +creating an account, a JSON file containing the Service Account's +credentials will be downloaded onto your machines. These credentials +are what rclone will use for authentication. + +To use a Service Account instead of OAuth2 token flow, enter the path +to your Service Account credentials at the `service_account_file` +prompt and rclone won't use the browser based authentication +flow. If you'd rather stuff the contents of the credentials file into +the rclone config file, you can set `service_account_credentials` with +the actual contents of the file instead, or set the equivalent +environment variable. + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +### Modified time ### + +Google google cloud storage stores md5sums natively and rclone stores +modification times as metadata on the object, under the "mtime" key in +RFC3339 format accurate to 1ns. + +Google Drive +----------------------------------------- + +Paths are specified as `drive:path` + +Drive paths may be as deep as required, eg `drive:directory/subdirectory`. + +The initial setup for drive involves getting a token from Google drive +which you need to do in your browser. `rclone config` walks you +through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +n/r/c/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value +[snip] +10 / Google Drive + \ "drive" +[snip] +Storage> drive +Google Application Client Id - leave blank normally. +client_id> +Google Application Client Secret - leave blank normally. +client_secret> +Scope that rclone should use when requesting access from drive. +Choose a number from below, or type in your own value + 1 / Full access all files, excluding Application Data Folder. + \ "drive" + 2 / Read-only access to file metadata and file contents. + \ "drive.readonly" + / Access to files created by rclone only. + 3 | These are visible in the drive website. + | File authorization is revoked when the user deauthorizes the app. + \ "drive.file" + / Allows read and write access to the Application Data folder. + 4 | This is not visible in the drive website. + \ "drive.appfolder" + / Allows read-only access to file metadata but + 5 | does not allow any access to read or download file content. + \ "drive.metadata.readonly" +scope> 1 +ID of the root folder - leave blank normally. Fill in to access "Computers" folders. (see docs). +root_folder_id> +Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. +service_account_file> +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine or Y didn't work +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +Configure this as a team drive? +y) Yes +n) No +y/n> n +-------------------- +[remote] +client_id = +client_secret = +scope = drive +root_folder_id = +service_account_file = +token = {"access_token":"XXX","token_type":"Bearer","refresh_token":"XXX","expiry":"2014-03-16T13:57:58.955387075Z"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Google if you use auto config mode. This only +runs from the moment it opens your browser to the moment you get back +the verification code. This is on `http://127.0.0.1:53682/` and this +it may require you to unblock it temporarily if you are running a host +firewall, or use manual mode. + +You can then use it like this, + +List directories in top level of your drive + + rclone lsd remote: + +List all the files in your drive + + rclone ls remote: + +To copy a local directory to a drive directory called backup + + rclone copy /home/source remote:backup + +### Scopes ### + +Rclone allows you to select which scope you would like for rclone to +use. This changes what type of token is granted to rclone. [The +scopes are defined +here.](https://developers.google.com/drive/v3/web/about-auth). + +The scope are + +#### drive #### + +This is the default scope and allows full access to all files, except +for the Application Data Folder (see below). + +Choose this one if you aren't sure. + +#### drive.readonly #### + +This allows read only access to all files. Files may be listed and +downloaded but not uploaded, renamed or deleted. + +#### drive.file #### + +With this scope rclone can read/view/modify only those files and +folders it creates. + +So if you uploaded files to drive via the web interface (or any other +means) they will not be visible to rclone. + +This can be useful if you are using rclone to backup data and you want +to be sure confidential data on your drive is not visible to rclone. + +Files created with this scope are visible in the web interface. + +#### drive.appfolder #### + +This gives rclone its own private area to store files. Rclone will +not be able to see any other files on your drive and you won't be able +to see rclone's files from the web interface either. + +#### drive.metadata.readonly #### + +This allows read only access to file names only. It does not allow +rclone to download or upload data, or rename or delete files or +directories. + +### Root folder ID ### + +You can set the `root_folder_id` for rclone. This is the directory +(identified by its `Folder ID`) that rclone considers to be a the root +of your drive. + +Normally you will leave this blank and rclone will determine the +correct root to use itself. + +However you can set this to restrict rclone to a specific folder +hierarchy or to access data within the "Computers" tab on the drive +web interface (where files from Google's Backup and Sync desktop +program go). + +In order to do this you will have to find the `Folder ID` of the +directory you wish rclone to display. This will be the last segment +of the URL when you open the relevant folder in the drive web +interface. + +So if the folder you want rclone to use has a URL which looks like +`https://drive.google.com/drive/folders/1XyfxxxxxxxxxxxxxxxxxxxxxxxxxKHCh` +in the browser, then you use `1XyfxxxxxxxxxxxxxxxxxxxxxxxxxKHCh` as +the `root_folder_id` in the config. + +**NB** folders under the "Computers" tab seem to be read only (drive +gives a 500 error) when using rclone. + +There doesn't appear to be an API to discover the folder IDs of the +"Computers" tab - please contact us if you know otherwise! + +Note also that rclone can't access any data under the "Backups" tab on +the google drive web interface yet. + +### Service Account support ### + +You can set up rclone with Google Drive in an unattended mode, +i.e. not tied to a specific end-user Google account. This is useful +when you want to synchronise files onto machines that don't have +actively logged-in users, for example build machines. + +To use a Service Account instead of OAuth2 token flow, enter the path +to your Service Account credentials at the `service_account_file` +prompt during `rclone config` and rclone won't use the browser based +authentication flow. If you'd rather stuff the contents of the +credentials file into the rclone config file, you can set +`service_account_credentials` with the actual contents of the file +instead, or set the equivalent environment variable. + +#### Use case - Google Apps/G-suite account and individual Drive #### + +Let's say that you are the administrator of a Google Apps (old) or +G-suite account. +The goal is to store data on an individual's Drive account, who IS +a member of the domain. +We'll call the domain **example.com**, and the user +**foo@example.com**. + +There's a few steps we need to go through to accomplish this: + +##### 1. Create a service account for example.com ##### + - To create a service account and obtain its credentials, go to the +[Google Developer Console](https://console.developers.google.com). + - You must have a project - create one if you don't. + - Then go to "IAM & admin" -> "Service Accounts". + - Use the "Create Credentials" button. Fill in "Service account name" +with something that identifies your client. "Role" can be empty. + - Tick "Furnish a new private key" - select "Key type JSON". + - Tick "Enable G Suite Domain-wide Delegation". This option makes +"impersonation" possible, as documented here: +[Delegating domain-wide authority to the service account](https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority) + - These credentials are what rclone will use for authentication. +If you ever need to remove access, press the "Delete service +account key" button. + +##### 2. Allowing API access to example.com Google Drive ##### + - Go to example.com's admin console + - Go into "Security" (or use the search bar) + - Select "Show more" and then "Advanced settings" + - Select "Manage API client access" in the "Authentication" section + - In the "Client Name" field enter the service account's +"Client ID" - this can be found in the Developer Console under +"IAM & Admin" -> "Service Accounts", then "View Client ID" for +the newly created service account. +It is a ~21 character numerical string. + - In the next field, "One or More API Scopes", enter +`https://www.googleapis.com/auth/drive` +to grant access to Google Drive specifically. + +##### 3. Configure rclone, assuming a new install ##### + +``` +rclone config + +n/s/q> n # New +name>gdrive # Gdrive is an example name +Storage> # Select the number shown for Google Drive +client_id> # Can be left blank +client_secret> # Can be left blank +scope> # Select your scope, 1 for example +root_folder_id> # Can be left blank +service_account_file> /home/foo/myJSONfile.json # This is where the JSON file goes! +y/n> # Auto config, y + +``` + +##### 4. Verify that it's working ##### + - `rclone -v --drive-impersonate foo@example.com lsf gdrive:backup` + - The arguments do: + - `-v` - verbose logging + - `--drive-impersonate foo@example.com` - this is what does +the magic, pretending to be user foo. + - `lsf` - list files in a parsing friendly way + - `gdrive:backup` - use the remote called gdrive, work in +the folder named backup. + +### Team drives ### + +If you want to configure the remote to point to a Google Team Drive +then answer `y` to the question `Configure this as a team drive?`. + +This will fetch the list of Team Drives from google and allow you to +configure which one you want to use. You can also type in a team +drive ID if you prefer. + +For example: + +``` +Configure this as a team drive? +y) Yes +n) No +y/n> y +Fetching team drive list... +Choose a number from below, or type in your own value + 1 / Rclone Test + \ "xxxxxxxxxxxxxxxxxxxx" + 2 / Rclone Test 2 + \ "yyyyyyyyyyyyyyyyyyyy" + 3 / Rclone Test 3 + \ "zzzzzzzzzzzzzzzzzzzz" +Enter a Team Drive ID> 1 +-------------------- +[remote] +client_id = +client_secret = +token = {"AccessToken":"xxxx.x.xxxxx_xxxxxxxxxxx_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","RefreshToken":"1/xxxxxxxxxxxxxxxx_xxxxxxxxxxxxxxxxxxxxxxxxxx","Expiry":"2014-03-16T13:57:58.955387075Z","Extra":null} +team_drive = xxxxxxxxxxxxxxxxxxxx +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +It does this by combining multiple `list` calls into a single API request. + +This works by combining many `'%s' in parents` filters into one expression. +To list the contents of directories a, b and c, the the following requests will be send by the regular `List` function: +``` +trashed=false and 'a' in parents +trashed=false and 'b' in parents +trashed=false and 'c' in parents +``` +These can now be combined into a single request: +``` +trashed=false and ('a' in parents or 'b' in parents or 'c' in parents) +``` + +The implementation of `ListR` will put up to 50 `parents` filters into one request. +It will use the `--checkers` value to specify the number of requests to run in parallel. + +In tests, these batch requests were up to 20x faster than the regular method. +Running the following command against different sized folders gives: +``` +rclone lsjson -vv -R --checkers=6 gdrive:folder +``` + +small folder (220 directories, 700 files): + +- without `--fast-list`: 38s +- with `--fast-list`: 10s + +large folder (10600 directories, 39000 files): + +- without `--fast-list`: 22:05 min +- with `--fast-list`: 58s + +### Modified time ### + +Google drive stores modification times accurate to 1 ms. + +### Revisions ### + +Google drive stores revisions of files. When you upload a change to +an existing file to google drive using rclone it will create a new +revision of that file. + +Revisions follow the standard google policy which at time of writing +was + + * They are deleted after 30 days or 100 revisions (whatever comes first). + * They do not count towards a user storage quota. + +### Deleting files ### + +By default rclone will send all files to the trash when deleting +files. If deleting them permanently is required then use the +`--drive-use-trash=false` flag, or set the equivalent environment +variable. + +### Emptying trash ### + +If you wish to empty your trash you can use the `rclone cleanup remote:` +command which will permanently delete all your trashed files. This command +does not take any path arguments. + +### Quota information ### + +To view your current quota you can use the `rclone about remote:` +command which will display your usage limit (quota), the usage in Google +Drive, the size of all files in the Trash and the space used by other +Google services such as Gmail. This command does not take any path +arguments. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --drive-acknowledge-abuse #### + +If downloading a file returns the error `This file has been identified +as malware or spam and cannot be downloaded` with the error code +`cannotDownloadAbusiveFile` then supply this flag to rclone to +indicate you acknowledge the risks of downloading the file and rclone +will download it anyway. + +#### --drive-auth-owner-only #### + +Only consider files owned by the authenticated user. + +#### --drive-chunk-size=SIZE #### + +Upload chunk size. Must a power of 2 >= 256k. Default value is 8 MB. + +Making this larger will improve performance, but note that each chunk +is buffered in memory one per transfer. + +Reducing this will reduce memory usage but decrease performance. + +#### --drive-formats #### + +Google documents can only be exported from Google drive. When rclone +downloads a Google doc it chooses a format to download depending upon +this setting. + +By default the formats are `docx,xlsx,pptx,svg` which are a sensible +default for an editable document. + +When choosing a format, rclone runs down the list provided in order +and chooses the first file format the doc can be exported as from the +list. If the file can't be exported to a format on the formats list, +then rclone will choose a format from the default list. + +If you prefer an archive copy then you might use `--drive-formats +pdf`, or if you prefer openoffice/libreoffice formats you might use +`--drive-formats ods,odt,odp`. + +Note that rclone adds the extension to the google doc, so if it is +calles `My Spreadsheet` on google docs, it will be exported as `My +Spreadsheet.xlsx` or `My Spreadsheet.pdf` etc. + +Here are the possible extensions with their corresponding mime types. + +| Extension | Mime Type | Description | +| --------- |-----------| ------------| +| csv | text/csv | Standard CSV format for Spreadsheets | +| doc | application/msword | Micosoft Office Document | +| docx | application/vnd.openxmlformats-officedocument.wordprocessingml.document | Microsoft Office Document | +| epub | application/epub+zip | E-book format | +| html | text/html | An HTML Document | +| jpg | image/jpeg | A JPEG Image File | +| odp | application/vnd.oasis.opendocument.presentation | Openoffice Presentation | +| ods | application/vnd.oasis.opendocument.spreadsheet | Openoffice Spreadsheet | +| ods | application/x-vnd.oasis.opendocument.spreadsheet | Openoffice Spreadsheet | +| odt | application/vnd.oasis.opendocument.text | Openoffice Document | +| pdf | application/pdf | Adobe PDF Format | +| png | image/png | PNG Image Format| +| pptx | application/vnd.openxmlformats-officedocument.presentationml.presentation | Microsoft Office Powerpoint | +| rtf | application/rtf | Rich Text Format | +| svg | image/svg+xml | Scalable Vector Graphics Format | +| tsv | text/tab-separated-values | Standard TSV format for spreadsheets | +| txt | text/plain | Plain Text | +| xls | application/vnd.ms-excel | Microsoft Office Spreadsheet | +| xlsx | application/vnd.openxmlformats-officedocument.spreadsheetml.sheet | Microsoft Office Spreadsheet | +| zip | application/zip | A ZIP file of HTML, Images CSS | + +#### --drive-alternate-export #### + +If this option is set this instructs rclone to use an alternate set of +export URLs for drive documents. Users have reported that the +official export URLs can't export large documents, whereas these +unofficial ones can. + +See rclone issue [#2243](https://github.com/ncw/rclone/issues/2243) for background, +[this google drive issue](https://issuetracker.google.com/issues/36761333) and +[this helpful post](https://www.labnol.org/internet/direct-links-for-google-drive/28356/). + +#### --drive-impersonate user #### + +When using a service account, this instructs rclone to impersonate the user passed in. + +#### --drive-keep-revision-forever #### + +Keeps new head revision of the file forever. + +#### --drive-list-chunk int #### + +Size of listing chunk 100-1000. 0 to disable. (default 1000) + +#### --drive-shared-with-me #### + +Instructs rclone to operate on your "Shared with me" folder (where +Google Drive lets you access the files and folders others have shared +with you). + +This works both with the "list" (lsd, lsl, etc) and the "copy" +commands (copy, sync, etc), and with all other commands too. + +#### --drive-skip-gdocs #### + +Skip google documents in all listings. If given, gdocs practically become invisible to rclone. + +#### --drive-trashed-only #### + +Only show files that are in the trash. This will show trashed files +in their original directory structure. + +#### --drive-upload-cutoff=SIZE #### + +File size cutoff for switching to chunked upload. Default is 8 MB. + +#### --drive-use-trash #### + +Controls whether files are sent to the trash or deleted +permanently. Defaults to true, namely sending files to the trash. Use +`--drive-use-trash=false` to delete files permanently instead. + +#### --drive-use-created-date #### + +Use the file creation date in place of the modification date. Defaults +to false. + +Useful when downloading data and you want the creation date used in +place of the last modified date. + +**WARNING**: This flag may have some unexpected consequences. + +When uploading to your drive all files will be overwritten unless they +haven't been modified since their creation. And the inverse will occur +while downloading. This side effect can be avoided by using the +`--checksum` flag. + +This feature was implemented to retain photos capture date as recorded +by google photos. You will first need to check the "Create a Google +Photos folder" option in your google drive settings. You can then copy +or move the photos locally and use the date the image was taken +(created) set as the modification date. + +### Limitations ### + +Drive has quite a lot of rate limiting. This causes rclone to be +limited to transferring about 2 files per second only. Individual +files may be transferred much faster at 100s of MBytes/s but lots of +small files can take a long time. + +Server side copies are also subject to a separate rate limit. If you +see User rate limit exceeded errors, wait at least 24 hours and retry. +You can disable server side copies with `--disable copy` to download +and upload the files if you prefer. + +#### Limitations of Google Docs #### + +Google docs will appear as size -1 in `rclone ls` and as size 0 in +anything which uses the VFS layer, eg `rclone mount`, `rclone serve`. + +This is because rclone can't find out the size of the Google docs +without downloading them. + +Google docs will transfer correctly with `rclone sync`, `rclone copy` +etc as rclone knows to ignore the size when doing the transfer. + +However an unfortunate consequence of this is that you can't download +Google docs using `rclone mount` - you will get a 0 sized file. If +you try again the doc may gain its correct size and be downloadable. + +### Duplicated files ### + +Sometimes, for no reason I've been able to track down, drive will +duplicate a file that rclone uploads. Drive unlike all the other +remotes can have duplicated files. + +Duplicated files cause problems with the syncing and you will see +messages in the log about duplicates. + +Use `rclone dedupe` to fix duplicated files. + +Note that this isn't just a problem with rclone, even Google Photos on +Android duplicates files on drive sometimes. + +### Rclone appears to be re-copying files it shouldn't ### + +The most likely cause of this is the duplicated file issue above - run +`rclone dedupe` and check your logs for duplicate object or directory +messages. + +### Making your own client_id ### + +When you use rclone with Google drive in its default configuration you +are using rclone's client_id. This is shared between all the rclone +users. There is a global rate limit on the number of queries per +second that each client_id can do set by Google. rclone already has a +high quota and I will continue to make sure it is high enough by +contacting Google. + +However you might find you get better performance making your own +client_id if you are a heavy user. Or you may not depending on exactly +how Google have been raising rclone's rate limit. + +Here is how to create your own Google Drive client ID for rclone: + +1. Log into the [Google API +Console](https://console.developers.google.com/) with your Google +account. It doesn't matter what Google account you use. (It need not +be the same account as the Google Drive you want to access) + +2. Select a project or create a new project. + +3. Under "ENABLE APIS AND SERVICES" search for "Drive", and enable the +then "Google Drive API". + +4. Click "Credentials" in the left-side panel (not "Create +credentials", which opens the wizard), then "Create credentials", then +"OAuth client ID". It will prompt you to set the OAuth consent screen +product name, if you haven't set one already. + +5. Choose an application type of "other", and click "Create". (the +default name is fine) + +6. It will show you a client ID and client secret. Use these values +in rclone config to add a new remote or edit an existing remote. + +(Thanks to @balazer on github for these instructions.) + +HTTP +------------------------------------------------- + +The HTTP remote is a read only remote for reading files of a +webserver. The webserver should provide file listings which rclone +will read and turn into a remote. This has been tested with common +webservers such as Apache/Nginx/Caddy and will likely work with file +listings from most web servers. (If it doesn't then please file an +issue, or send a pull request!) + +Paths are specified as `remote:` or `remote:path/to/dir`. + +Here is an example of how to make a remote called `remote`. First +run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / FTP Connection + \ "ftp" + 7 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 8 / Google Drive + \ "drive" + 9 / Hubic + \ "hubic" +10 / Local Disk + \ "local" +11 / Microsoft OneDrive + \ "onedrive" +12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +13 / SSH/SFTP Connection + \ "sftp" +14 / Yandex Disk + \ "yandex" +15 / http Connection + \ "http" +Storage> http +URL of http host to connect to +Choose a number from below, or type in your own value + 1 / Connect to example.com + \ "https://example.com" +url> https://beta.rclone.org +Remote config +-------------------- +[remote] +url = https://beta.rclone.org +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +Current remotes: + +Name Type +==== ==== +remote http + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> q +``` + +This remote is called `remote` and can now be used like this + +See all the top level directories + + rclone lsd remote: + +List the contents of a directory + + rclone ls remote:directory + +Sync the remote `directory` to `/home/local/directory`, deleting any excess files. + + rclone sync remote:directory /home/local/directory + +### Read only ### + +This remote is read only - you can't upload files to an HTTP server. + +### Modified time ### + +Most HTTP servers store time accurate to 1 second. + +### Checksum ### + +No checksums are stored. + +### Usage without a config file ### + +Since the http remote only has one config parameter it is easy to use +without a config file: + + rclone lsd --http-url https://beta.rclone.org :http: + +Hubic +----------------------------------------- + +Paths are specified as `remote:path` + +Paths are specified as `remote:container` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, eg `remote:container/path/to/dir`. + +The initial setup for Hubic involves getting a token from Hubic which +you need to do in your browser. `rclone config` walks you through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +n) New remote +s) Set configuration password +n/s> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" +10 / Microsoft OneDrive + \ "onedrive" +11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +12 / SSH/SFTP Connection + \ "sftp" +13 / Yandex Disk + \ "yandex" +Storage> 8 +Hubic Client Id - leave blank normally. +client_id> +Hubic Client Secret - leave blank normally. +client_secret> +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +client_id = +client_secret = +token = {"access_token":"XXXXXX"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Hubic. This only runs from the moment it opens +your browser to the moment you get back the verification code. This +is on `http://127.0.0.1:53682/` and this it may require you to unblock +it temporarily if you are running a host firewall. + +Once configured you can then use `rclone` like this, + +List containers in the top level of your Hubic + + rclone lsd remote: + +List all the files in your Hubic + + rclone ls remote: + +To copy a local directory to an Hubic directory called backup + + rclone copy /home/source remote:backup + +If you want the directory to be visible in the official *Hubic +browser*, you need to copy your files to the `default` directory + + rclone copy /home/source remote:default/backup + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +### Modified time ### + +The modified time is stored as metadata on the object as +`X-Object-Meta-Mtime` as floating point since the epoch accurate to 1 +ns. + +This is a defacto standard (used in the official python-swiftclient +amongst others) for storing the modification time for an object. + +Note that Hubic wraps the Swift backend, so most of the properties of +are the same. + +### Limitations ### + +This uses the normal OpenStack Swift mechanism to refresh the Swift +API credentials and ignores the expires field returned by the Hubic +API. + +The Swift API doesn't return a correct MD5SUM for segmented files +(Dynamic or Static Large Objects) so rclone won't check or use the +MD5SUM for these. + +Jottacloud +----------------------------------------- + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +To configure Jottacloud you will need to enter your username and password and select a mountpoint. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value +[snip] +13 / JottaCloud + \ "jottacloud" +[snip] +Storage> jottacloud +User Name +Enter a string value. Press Enter for the default (""). +user> user +Password. +y) Yes type in my own password +g) Generate random password +n) No leave this optional password blank +y/g/n> y +Enter the password: +password: +Confirm the password: +password: +The mountpoint to use. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 1 / Will be synced by the official client. + \ "Sync" + 2 / Archive + \ "Archive" +mountpoint> Archive +Remote config +-------------------- +[remote] +type = jottacloud +user = user +pass = *** ENCRYPTED *** +mountpoint = Archive +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` +Once configured you can then use `rclone` like this, + +List directories in top level of your Jottacloud + + rclone lsd remote: + +List all the files in your Jottacloud + + rclone ls remote: + +To copy a local directory to an Jottacloud directory called backup + + rclone copy /home/source remote:backup + + +### Modified time and hashes ### + +Jottacloud allows modification times to be set on objects accurate to 1 +second. These will be used to detect whether objects need syncing or +not. + +Jottacloud supports MD5 type hashes, so you can use the `--checksum` +flag. + +Note that Jottacloud requires the MD5 hash before upload so if the +source does not have an MD5 checksum then the file will be cached +temporarily on disk (wherever the `TMPDIR` environment variable points +to) before it is uploaded. Small files will be cached in memory - see +the `--jottacloud-md5-memory-limit` flag. + +### Deleting files ### + +Any files you delete with rclone will end up in the trash. Due to a lack of API documentation emptying the trash is currently only possible via the Jottacloud website. + +### Versions ### + +Jottacloud supports file versioning. When rclone uploads a new version of a file it creates a new version of it. Currently rclone only supports retrieving the current version but older versions can be accessed via the Jottacloud Website. + +### Limitations ### + +Note that Jottacloud is case insensitive so you can't have a file called +"Hello.doc" and one called "hello.doc". + +There are quite a few characters that can't be in Jottacloud file names. Rclone will map these names to and from an identical looking unicode equivalent. For example if a file has a ? in it will be mapped to ？ instead. + +Jottacloud only supports filenames up to 255 characters in length. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --jottacloud-md5-memory-limit SizeSuffix + +Files bigger than this will be cached on disk to calculate the MD5 if +required. (default 10M) + +### Troubleshooting ### + +Jottacloud exhibits some inconsistent behaviours regarding deleted files and folders which may cause Copy, Move and DirMove operations to previously deleted paths to fail. Emptying the trash should help in such cases. + +Mega +----------------------------------------- + +[Mega](https://mega.nz/) is a cloud storage and file hosting service +known for its security feature where all files are encrypted locally +before they are uploaded. This prevents anyone (including employees of +Mega) from accessing the files without knowledge of the key used for +encryption. + +This is an rclone backend for Mega which supports the file transfer +features of Mega using the same client side encryption. + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" +[snip] +14 / Mega + \ "mega" +[snip] +23 / http Connection + \ "http" +Storage> mega +User name +user> you@example.com +Password. +y) Yes type in my own password +g) Generate random password +n) No leave this optional password blank +y/g/n> y +Enter the password: +password: +Confirm the password: +password: +Remote config +-------------------- +[remote] +type = mega +user = you@example.com +pass = *** ENCRYPTED *** +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Once configured you can then use `rclone` like this, + +List directories in top level of your Mega + + rclone lsd remote: + +List all the files in your Mega + + rclone ls remote: + +To copy a local directory to an Mega directory called backup + + rclone copy /home/source remote:backup + +### Modified time and hashes ### + +Mega does not support modification times or hashes yet. + +### Duplicated files ### + +Mega can have two files with exactly the same name and path (unlike a +normal file system). + +Duplicated files cause problems with the syncing and you will see +messages in the log about duplicates. + +Use `rclone dedupe` to fix duplicated files. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --mega-debug #### + +If this flag is set (along with `-vv`) it will print further debugging +information from the mega backend. + +#### --mega-hard-delete #### + +Normally the mega backend will put all deletions into the trash rather +than permanently deleting them. If you specify this flag (or set it +in the advanced config) then rclone will permanently delete objects +instead. + +### Limitations ### + +This backend uses the [go-mega go +library](https://github.com/t3rm1n4l/go-mega) which is an opensource +go library implementing the Mega API. There doesn't appear to be any +documentation for the mega protocol beyond the [mega C++ +SDK](https://github.com/meganz/sdk) source code so there are likely +quite a few errors still remaining in this library. + +Mega allows duplicate files which may confuse rclone. + +Microsoft Azure Blob Storage +----------------------------------------- + +Paths are specified as `remote:container` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, eg +`remote:container/path/to/dir`. + +Here is an example of making a Microsoft Azure Blob Storage +configuration. For a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Box + \ "box" + 5 / Dropbox + \ "dropbox" + 6 / Encrypt/Decrypt a remote + \ "crypt" + 7 / FTP Connection + \ "ftp" + 8 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 9 / Google Drive + \ "drive" +10 / Hubic + \ "hubic" +11 / Local Disk + \ "local" +12 / Microsoft Azure Blob Storage + \ "azureblob" +13 / Microsoft OneDrive + \ "onedrive" +14 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +15 / SSH/SFTP Connection + \ "sftp" +16 / Yandex Disk + \ "yandex" +17 / http Connection + \ "http" +Storage> azureblob +Storage Account Name +account> account_name +Storage Account Key +key> base64encodedkey== +Endpoint for the service - leave blank normally. +endpoint> +Remote config +-------------------- +[remote] +account = account_name +key = base64encodedkey== +endpoint = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +See all containers + + rclone lsd remote: + +Make a new container + + rclone mkdir remote:container + +List the contents of a container + + rclone ls remote:container + +Sync `/home/local/directory` to the remote container, deleting any excess +files in the container. + + rclone sync /home/local/directory remote:container + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +### Modified time ### + +The modified time is stored as metadata on the object with the `mtime` +key. It is stored using RFC3339 Format time with nanosecond +precision. The metadata is supplied during directory listings so +there is no overhead to using it. + +### Hashes ### + +MD5 hashes are stored with blobs. However blobs that were uploaded in +chunks only have an MD5 if the source remote was capable of MD5 +hashes, eg the local disk. + +### Authenticating with Azure Blob Storage + +Rclone has 3 ways of authenticating with Azure Blob Storage: + +#### Account and Key + +This is the most straight forward and least flexible way. Just fill in the `account` and `key` lines and leave the rest blank. + +#### SAS URL + +This can be an account level SAS URL or container level SAS URL + +To use it leave `account`, `key` blank and fill in `sas_url`. + +Account level SAS URL or container level SAS URL can be obtained from Azure portal or Azure Storage Explorer. +To get a container level SAS URL right click on a container in the Azure Blob explorer in the Azure portal. + +If You use container level SAS URL, rclone operations are permitted only on particular container, eg + + rclone ls azureblob:container or rclone ls azureblob: + +Since container name already exists in SAS URL, you can leave it empty as well. + +However these will not work + + rclone lsd azureblob: + rclone ls azureblob:othercontainer + +This would be useful for temporarily allowing third parties access to a single container or putting credentials into an untrusted environment. + +### Multipart uploads ### + +Rclone supports multipart uploads with Azure Blob storage. Files +bigger than 256MB will be uploaded using chunked upload by default. + +The files will be uploaded in parallel in 4MB chunks (by default). +Note that these chunks are buffered in memory and there may be up to +`--transfers` of them being uploaded at once. + +Files can't be split into more than 50,000 chunks so by default, so +the largest file that can be uploaded with 4MB chunk size is 195GB. +Above this rclone will double the chunk size until it creates less +than 50,000 chunks. By default this will mean a maximum file size of +3.2TB can be uploaded. This can be raised to 5TB using +`--azureblob-chunk-size 100M`. + +Note that rclone doesn't commit the block list until the end of the +upload which means that there is a limit of 9.5TB of multipart uploads +in progress as Azure won't allow more than that amount of uncommitted +blocks. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --azureblob-upload-cutoff=SIZE #### + +Cutoff for switching to chunked upload - must be <= 256MB. The default +is 256MB. + +#### --azureblob-chunk-size=SIZE #### + +Upload chunk size. Default 4MB. Note that this is stored in memory +and there may be up to `--transfers` chunks stored at once in memory. +This can be at most 100MB. + +#### --azureblob-access-tier=Hot/Cool/Archive #### + +Azure storage supports blob tiering, you can configure tier in advanced +settings or supply flag while performing data transfer operations. +If there is no `access tier` specified, rclone doesn't apply any tier. +rclone performs `Set Tier` operation on blobs while uploading, if objects +are not modified, specifying `access tier` to new one will have no effect. +If blobs are in `archive tier` at remote, trying to perform data transfer +operations from remote will not be allowed. User should first restore by +tiering blob to `Hot` or `Cool`. + +### Limitations ### + +MD5 sums are only uploaded with chunked files if the source has an MD5 +sum. This will always be the case for a local to azure copy. + +Microsoft OneDrive +----------------------------------------- + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +The initial setup for OneDrive involves getting a token from +Microsoft which you need to do in your browser. `rclone config` walks +you through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +n/s> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" +10 / Microsoft OneDrive + \ "onedrive" +11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +12 / SSH/SFTP Connection + \ "sftp" +13 / Yandex Disk + \ "yandex" +Storage> 10 +Microsoft App Client Id - leave blank normally. +client_id> +Microsoft App Client Secret - leave blank normally. +client_secret> +Remote config +Choose OneDrive account type? + * Say b for a OneDrive business account + * Say p for a personal OneDrive account +b) Business +p) Personal +b/p> p +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +client_id = +client_secret = +token = {"access_token":"XXXXXX"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Microsoft. This only runs from the moment it +opens your browser to the moment you get back the verification +code. This is on `http://127.0.0.1:53682/` and this it may require +you to unblock it temporarily if you are running a host firewall. + +Once configured you can then use `rclone` like this, + +List directories in top level of your OneDrive + + rclone lsd remote: + +List all the files in your OneDrive + + rclone ls remote: + +To copy a local directory to an OneDrive directory called backup + + rclone copy /home/source remote:backup + +### OneDrive for Business ### + +There is additional support for OneDrive for Business. +Select "b" when ask +``` +Choose OneDrive account type? + * Say b for a OneDrive business account + * Say p for a personal OneDrive account +b) Business +p) Personal +b/p> +``` +After that rclone requires an authentication of your account. The application will first authenticate your account, then query the OneDrive resource URL +and do a second (silent) authentication for this resource URL. + +### Modified time and hashes ### + +OneDrive allows modification times to be set on objects accurate to 1 +second. These will be used to detect whether objects need syncing or +not. + +OneDrive personal supports SHA1 type hashes. OneDrive for business and +Sharepoint Server support +[QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash). + +For all types of OneDrive you can use the `--checksum` flag. + +### Deleting files ### + +Any files you delete with rclone will end up in the trash. Microsoft +doesn't provide an API to permanently delete files, nor to empty the +trash, so you will have to do that with one of Microsoft's apps or via +the OneDrive website. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --onedrive-chunk-size=SIZE #### + +Above this size files will be chunked - must be multiple of 320k. The +default is 10MB. Note that the chunks will be buffered into memory. + +### Limitations ### + +Note that OneDrive is case insensitive so you can't have a +file called "Hello.doc" and one called "hello.doc". + +There are quite a few characters that can't be in OneDrive file +names. These can't occur on Windows platforms, but on non-Windows +platforms they are common. Rclone will map these names to and from an +identical looking unicode equivalent. For example if a file has a `?` +in it will be mapped to `？` instead. + +The largest allowed file size is 10GiB (10,737,418,240 bytes). + +### Versioning issue ### + +Every change in OneDrive causes the service to create a new version. +This counts against a users quota. +For example changing the modification time of a file creates a second +version, so the file is using twice the space. + +The `copy` is the only rclone command affected by this as we copy +the file and then afterwards set the modification time to match the +source file. + +User [Weropol](https://github.com/Weropol) has found a method to disable +versioning on OneDrive + +1. Open the settings menu by clicking on the gear symbol at the top of the OneDrive Business page. +2. Click Site settings. +3. Once on the Site settings page, navigate to Site Administration > Site libraries and lists. +4. Click Customize "Documents". +5. Click General Settings > Versioning Settings. +6. Under Document Version History select the option No versioning. +Note: This will disable the creation of new file versions, but will not remove any previous versions. Your documents are safe. +7. Apply the changes by clicking OK. +8. Use rclone to upload or modify files. (I also use the --no-update-modtime flag) +9. Restore the versioning settings after using rclone. (Optional) + +### Troubleshooting ### + +``` +Error: access_denied +Code: AADSTS65005 +Description: Using application 'rclone' is currently not supported for your organization [YOUR_ORGANIZATION] because it is in an unmanaged state. An administrator needs to claim ownership of the company by DNS validation of [YOUR_ORGANIZATION] before the application rclone can be provisioned. +``` + +This means that rclone can't use the OneDrive for Business API with your account. You can't do much about it, maybe write an email to your admins. + +However, there are other ways to interact with your OneDrive account. Have a look at the webdav backend: https://rclone.org/webdav/#sharepoint + +OpenDrive +------------------------------------ + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +n) New remote +d) Delete remote +q) Quit config +e/n/d/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" +10 / OpenDrive + \ "opendrive" +11 / Microsoft OneDrive + \ "onedrive" +12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +13 / SSH/SFTP Connection + \ "sftp" +14 / Yandex Disk + \ "yandex" +Storage> 10 +Username +username> +Password +y) Yes type in my own password +g) Generate random password +y/g> y +Enter the password: +password: +Confirm the password: +password: +-------------------- +[remote] +username = +password = *** ENCRYPTED *** +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +List directories in top level of your OpenDrive + + rclone lsd remote: + +List all the files in your OpenDrive + + rclone ls remote: + +To copy a local directory to an OpenDrive directory called backup + + rclone copy /home/source remote:backup + +### Modified time and MD5SUMs ### + +OpenDrive allows modification times to be set on objects accurate to 1 +second. These will be used to detect whether objects need syncing or +not. + +### Deleting files ### + +Any files you delete with rclone will end up in the trash. Amazon +don't provide an API to permanently delete files, nor to empty the +trash, so you will have to do that with one of Amazon's apps or via +the OpenDrive website. As of November 17, 2016, files are +automatically deleted by Amazon from the trash after 30 days. + +### Limitations ### + +Note that OpenDrive is case insensitive so you can't have a +file called "Hello.doc" and one called "hello.doc". + +There are quite a few characters that can't be in OpenDrive file +names. These can't occur on Windows platforms, but on non-Windows +platforms they are common. Rclone will map these names to and from an +identical looking unicode equivalent. For example if a file has a `?` +in it will be mapped to `？` instead. + +QingStor +--------------------------------------- + +Paths are specified as `remote:bucket` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, eg `remote:bucket/path/to/dir`. + +Here is an example of making an QingStor configuration. First run + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found - make a new one +n) New remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +n/r/c/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / FTP Connection + \ "ftp" + 7 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 8 / Google Drive + \ "drive" + 9 / Hubic + \ "hubic" +10 / Local Disk + \ "local" +11 / Microsoft OneDrive + \ "onedrive" +12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +13 / QingStor Object Storage + \ "qingstor" +14 / SSH/SFTP Connection + \ "sftp" +15 / Yandex Disk + \ "yandex" +Storage> 13 +Get QingStor credentials from runtime. Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own value + 1 / Enter QingStor credentials in the next step + \ "false" + 2 / Get QingStor credentials from the environment (env vars or IAM) + \ "true" +env_auth> 1 +QingStor Access Key ID - leave blank for anonymous access or runtime credentials. +access_key_id> access_key +QingStor Secret Access Key (password) - leave blank for anonymous access or runtime credentials. +secret_access_key> secret_key +Enter a endpoint URL to connection QingStor API. +Leave blank will use the default value "https://qingstor.com:443" +endpoint> +Zone connect to. Default is "pek3a". +Choose a number from below, or type in your own value + / The Beijing (China) Three Zone + 1 | Needs location constraint pek3a. + \ "pek3a" + / The Shanghai (China) First Zone + 2 | Needs location constraint sh1a. + \ "sh1a" +zone> 1 +Number of connnection retry. +Leave blank will use the default value "3". +connection_retries> +Remote config +-------------------- +[remote] +env_auth = false +access_key_id = access_key +secret_access_key = secret_key +endpoint = +zone = pek3a +connection_retries = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This remote is called `remote` and can now be used like this + +See all buckets + + rclone lsd remote: + +Make a new bucket + + rclone mkdir remote:bucket + +List the contents of a bucket + + rclone ls remote:bucket + +Sync `/home/local/directory` to the remote bucket, deleting any excess +files in the bucket. + + rclone sync /home/local/directory remote:bucket + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +### Multipart uploads ### + +rclone supports multipart uploads with QingStor which means that it can +upload files bigger than 5GB. Note that files uploaded with multipart +upload don't have an MD5SUM. + +### Buckets and Zone ### + +With QingStor you can list buckets (`rclone lsd`) using any zone, +but you can only access the content of a bucket from the zone it was +created in. If you attempt to access a bucket from the wrong zone, +you will get an error, `incorrect zone, the bucket is not in 'XXX' +zone`. + +### Authentication ### + +There are two ways to supply `rclone` with a set of QingStor +credentials. In order of precedence: + + - Directly in the rclone configuration file (as configured by `rclone config`) + - set `access_key_id` and `secret_access_key` + - Runtime configuration: + - set `env_auth` to `true` in the config file + - Exporting the following environment variables before running `rclone` + - Access Key ID: `QS_ACCESS_KEY_ID` or `QS_ACCESS_KEY` + - Secret Access Key: `QS_SECRET_ACCESS_KEY` or `QS_SECRET_KEY` + +Swift +---------------------------------------- + +Swift refers to [Openstack Object Storage](https://docs.openstack.org/swift/latest/). +Commercial implementations of that being: + + * [Rackspace Cloud Files](https://www.rackspace.com/cloud/files/) + * [Memset Memstore](https://www.memset.com/cloud/storage/) + * [OVH Object Storage](https://www.ovh.co.uk/public-cloud/storage/object-storage/) + * [Oracle Cloud Storage](https://cloud.oracle.com/storage-opc) + * [IBM Bluemix Cloud ObjectStorage Swift](https://console.bluemix.net/docs/infrastructure/objectstorage-swift/index.html) + +Paths are specified as `remote:container` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, eg `remote:container/path/to/dir`. + +Here is an example of making a swift configuration. First run + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Box + \ "box" + 5 / Cache a remote + \ "cache" + 6 / Dropbox + \ "dropbox" + 7 / Encrypt/Decrypt a remote + \ "crypt" + 8 / FTP Connection + \ "ftp" + 9 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" +10 / Google Drive + \ "drive" +11 / Hubic + \ "hubic" +12 / Local Disk + \ "local" +13 / Microsoft Azure Blob Storage + \ "azureblob" +14 / Microsoft OneDrive + \ "onedrive" +15 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +16 / Pcloud + \ "pcloud" +17 / QingCloud Object Storage + \ "qingstor" +18 / SSH/SFTP Connection + \ "sftp" +19 / Webdav + \ "webdav" +20 / Yandex Disk + \ "yandex" +21 / http Connection + \ "http" +Storage> swift +Get swift credentials from environment variables in standard OpenStack form. +Choose a number from below, or type in your own value + 1 / Enter swift credentials in the next step + \ "false" + 2 / Get swift credentials from environment vars. Leave other fields blank if using this. + \ "true" +env_auth> true +User name to log in (OS_USERNAME). +user> +API key or password (OS_PASSWORD). +key> +Authentication URL for server (OS_AUTH_URL). +Choose a number from below, or type in your own value + 1 / Rackspace US + \ "https://auth.api.rackspacecloud.com/v1.0" + 2 / Rackspace UK + \ "https://lon.auth.api.rackspacecloud.com/v1.0" + 3 / Rackspace v2 + \ "https://identity.api.rackspacecloud.com/v2.0" + 4 / Memset Memstore UK + \ "https://auth.storage.memset.com/v1.0" + 5 / Memset Memstore UK v2 + \ "https://auth.storage.memset.com/v2.0" + 6 / OVH + \ "https://auth.cloud.ovh.net/v2.0" +auth> +User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID). +user_id> +User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME) +domain> +Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME) +tenant> +Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID) +tenant_id> +Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME) +tenant_domain> +Region name - optional (OS_REGION_NAME) +region> +Storage URL - optional (OS_STORAGE_URL) +storage_url> +Auth Token from alternate authentication - optional (OS_AUTH_TOKEN) +auth_token> +AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION) +auth_version> +Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) +Choose a number from below, or type in your own value + 1 / Public (default, choose this if not sure) + \ "public" + 2 / Internal (use internal service net) + \ "internal" + 3 / Admin + \ "admin" +endpoint_type> +Remote config +-------------------- +[test] +env_auth = true +user = +key = +auth = +user_id = +domain = +tenant = +tenant_id = +tenant_domain = +region = +storage_url = +auth_token = +auth_version = +endpoint_type = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This remote is called `remote` and can now be used like this + +See all containers + + rclone lsd remote: + +Make a new container + + rclone mkdir remote:container + +List the contents of a container + + rclone ls remote:container + +Sync `/home/local/directory` to the remote container, deleting any +excess files in the container. + + rclone sync /home/local/directory remote:container + +### Configuration from an Openstack credentials file ### + +An Opentstack credentials file typically looks something something +like this (without the comments) + +``` +export OS_AUTH_URL=https://a.provider.net/v2.0 +export OS_TENANT_ID=ffffffffffffffffffffffffffffffff +export OS_TENANT_NAME="1234567890123456" +export OS_USERNAME="123abc567xy" +echo "Please enter your OpenStack Password: " +read -sr OS_PASSWORD_INPUT +export OS_PASSWORD=$OS_PASSWORD_INPUT +export OS_REGION_NAME="SBG1" +if [ -z "$OS_REGION_NAME" ]; then unset OS_REGION_NAME; fi +``` + +The config file needs to look something like this where `$OS_USERNAME` +represents the value of the `OS_USERNAME` variable - `123abc567xy` in +the example above. + +``` +[remote] +type = swift +user = $OS_USERNAME +key = $OS_PASSWORD +auth = $OS_AUTH_URL +tenant = $OS_TENANT_NAME +``` + +Note that you may (or may not) need to set `region` too - try without first. + +### Configuration from the environment ### + +If you prefer you can configure rclone to use swift using a standard +set of OpenStack environment variables. + +When you run through the config, make sure you choose `true` for +`env_auth` and leave everything else blank. + +rclone will then set any empty config parameters from the environment +using standard OpenStack environment variables. There is [a list of +the +variables](https://godoc.org/github.com/ncw/swift#Connection.ApplyEnvironment) +in the docs for the swift library. + +### Using an alternate authentication method ### + +If your OpenStack installation uses a non-standard authentication method +that might not be yet supported by rclone or the underlying swift library, +you can authenticate externally (e.g. calling manually the `openstack` +commands to get a token). Then, you just need to pass the two +configuration variables ``auth_token`` and ``storage_url``. +If they are both provided, the other variables are ignored. rclone will +not try to authenticate but instead assume it is already authenticated +and use these two variables to access the OpenStack installation. + +#### Using rclone without a config file #### + +You can use rclone with swift without a config file, if desired, like +this: + +``` +source openstack-credentials-file +export RCLONE_CONFIG_MYREMOTE_TYPE=swift +export RCLONE_CONFIG_MYREMOTE_ENV_AUTH=true +rclone lsd myremote: +``` + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +### --update and --use-server-modtime ### + +As noted below, the modified time is stored on metadata on the object. It is +used by default for all operations that require checking the time a file was +last updated. It allows rclone to treat the remote more like a true filesystem, +but it is inefficient because it requires an extra API call to retrieve the +metadata. + +For many operations, the time the object was last uploaded to the remote is +sufficient to determine if it is "dirty". By using `--update` along with +`--use-server-modtime`, you can avoid the extra API call and simply upload +files whose local modtime is newer than the time it was last uploaded. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --swift-storage-policy=STRING #### +Apply the specified storage policy when creating a new container. The policy +cannot be changed afterwards. The allowed configuration values and their +meaning depend on your Swift storage provider. + +#### --swift-chunk-size=SIZE #### + +Above this size files will be chunked into a _segments container. The +default for this is 5GB which is its maximum value. + +### Modified time ### + +The modified time is stored as metadata on the object as +`X-Object-Meta-Mtime` as floating point since the epoch accurate to 1 +ns. + +This is a defacto standard (used in the official python-swiftclient +amongst others) for storing the modification time for an object. + +### Limitations ### + +The Swift API doesn't return a correct MD5SUM for segmented files +(Dynamic or Static Large Objects) so rclone won't check or use the +MD5SUM for these. + +### Troubleshooting ### + +#### Rclone gives Failed to create file system for "remote:": Bad Request #### + +Due to an oddity of the underlying swift library, it gives a "Bad +Request" error rather than a more sensible error when the +authentication fails for Swift. + +So this most likely means your username / password is wrong. You can +investigate further with the `--dump-bodies` flag. + +This may also be caused by specifying the region when you shouldn't +have (eg OVH). + +#### Rclone gives Failed to create file system: Response didn't have storage storage url and auth token #### + +This is most likely caused by forgetting to specify your tenant when +setting up a swift remote. + +pCloud +----------------------------------------- + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +The initial setup for pCloud involves getting a token from pCloud which you +need to do in your browser. `rclone config` walks you through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Box + \ "box" + 5 / Dropbox + \ "dropbox" + 6 / Encrypt/Decrypt a remote + \ "crypt" + 7 / FTP Connection + \ "ftp" + 8 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 9 / Google Drive + \ "drive" +10 / Hubic + \ "hubic" +11 / Local Disk + \ "local" +12 / Microsoft Azure Blob Storage + \ "azureblob" +13 / Microsoft OneDrive + \ "onedrive" +14 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +15 / Pcloud + \ "pcloud" +16 / QingCloud Object Storage + \ "qingstor" +17 / SSH/SFTP Connection + \ "sftp" +18 / Yandex Disk + \ "yandex" +19 / http Connection + \ "http" +Storage> pcloud +Pcloud App Client Id - leave blank normally. +client_id> +Pcloud App Client Secret - leave blank normally. +client_secret> +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +client_id = +client_secret = +token = {"access_token":"XXX","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from pCloud. This only runs from the moment it opens +your browser to the moment you get back the verification code. This +is on `http://127.0.0.1:53682/` and this it may require you to unblock +it temporarily if you are running a host firewall. + +Once configured you can then use `rclone` like this, + +List directories in top level of your pCloud + + rclone lsd remote: + +List all the files in your pCloud + + rclone ls remote: + +To copy a local directory to an pCloud directory called backup + + rclone copy /home/source remote:backup + +### Modified time and hashes ### + +pCloud allows modification times to be set on objects accurate to 1 +second. These will be used to detect whether objects need syncing or +not. In order to set a Modification time pCloud requires the object +be re-uploaded. + +pCloud supports MD5 and SHA1 type hashes, so you can use the +`--checksum` flag. + +### Deleting files ### + +Deleted files will be moved to the trash. Your subscription level +will determine how long items stay in the trash. `rclone cleanup` can +be used to empty the trash. + +SFTP +---------------------------------------- + +SFTP is the [Secure (or SSH) File Transfer +Protocol](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol). + +SFTP runs over SSH v2 and is installed as standard with most modern +SSH installations. + +Paths are specified as `remote:path`. If the path does not begin with +a `/` it is relative to the home directory of the user. An empty path +`remote:` refers to the user's home directory. + +Note that some SFTP servers will need the leading `/` - Synology is a +good example of this. + +Here is an example of making an SFTP configuration. First run + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / FTP Connection + \ "ftp" + 7 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 8 / Google Drive + \ "drive" + 9 / Hubic + \ "hubic" +10 / Local Disk + \ "local" +11 / Microsoft OneDrive + \ "onedrive" +12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +13 / SSH/SFTP Connection + \ "sftp" +14 / Yandex Disk + \ "yandex" +15 / http Connection + \ "http" +Storage> sftp +SSH host to connect to +Choose a number from below, or type in your own value + 1 / Connect to example.com + \ "example.com" +host> example.com +SSH username, leave blank for current username, ncw +user> sftpuser +SSH port, leave blank to use default (22) +port> +SSH password, leave blank to use ssh-agent. +y) Yes type in my own password +g) Generate random password +n) No leave this optional password blank +y/g/n> n +Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent. +key_file> +Remote config +-------------------- +[remote] +host = example.com +user = sftpuser +port = +pass = +key_file = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This remote is called `remote` and can now be used like this: + +See all directories in the home directory + + rclone lsd remote: + +Make a new directory + + rclone mkdir remote:path/to/directory + +List the contents of a directory + + rclone ls remote:path/to/directory + +Sync `/home/local/directory` to the remote directory, deleting any +excess files in the directory. + + rclone sync /home/local/directory remote:directory + +### SSH Authentication ### + +The SFTP remote supports three authentication methods: + + * Password + * Key file + * ssh-agent + +Key files should be unencrypted PEM-encoded private key files. For +instance `/home/$USER/.ssh/id_rsa`. + +If you don't specify `pass` or `key_file` then rclone will attempt to +contact an ssh-agent. + +If you set the `--sftp-ask-password` option, rclone will prompt for a +password when needed and no password has been configured. + +### ssh-agent on macOS ### + +Note that there seem to be various problems with using an ssh-agent on +macOS due to recent changes in the OS. The most effective work-around +seems to be to start an ssh-agent in each session, eg + + eval `ssh-agent -s` && ssh-add -A + +And then at the end of the session + + eval `ssh-agent -k` + +These commands can be used in scripts of course. + +### Specific options ### + +Here are the command line options specific to this remote. + +#### --sftp-ask-password #### + +Ask for the SFTP password if needed when no password has been configured. + +#### --ssh-path-override #### + +Override path used by SSH connection. Allows checksum calculation when +SFTP and SSH paths are different. This issue affects among others Synology +NAS boxes. + +Shared folders can be found in directories representing volumes + + rclone sync /home/local/directory remote:/directory --ssh-path-override /volume2/directory + +Home directory can be found in a shared folder called `homes` + + rclone sync /home/local/directory remote:/home/directory --ssh-path-override /volume1/homes/USER/directory + +### Modified time ### + +Modified times are stored on the server to 1 second precision. + +Modified times are used in syncing and are fully supported. + +Some SFTP servers disable setting/modifying the file modification time after +upload (for example, certain configurations of ProFTPd with mod_sftp). If you +are using one of these servers, you can set the option `set_modtime = false` in +your RClone backend configuration to disable this behaviour. + +### Limitations ### + +SFTP supports checksums if the same login has shell access and `md5sum` +or `sha1sum` as well as `echo` are in the remote's PATH. +This remote checksumming (file hashing) is recommended and enabled by default. +Disabling the checksumming may be required if you are connecting to SFTP servers +which are not under your control, and to which the execution of remote commands +is prohibited. Set the configuration option `disable_hashcheck` to `true` to +disable checksumming. + +Note that some SFTP servers (eg Synology) the paths are different for +SSH and SFTP so the hashes can't be calculated properly. For them +using `disable_hashcheck` is a good idea. + +The only ssh agent supported under Windows is Putty's pageant. + +The Go SSH library disables the use of the aes128-cbc cipher by +default, due to security concerns. This can be re-enabled on a +per-connection basis by setting the `use_insecure_cipher` setting in +the configuration file to `true`. Further details on the insecurity of +this cipher can be found [in this paper] +(http://www.isg.rhul.ac.uk/~kp/SandPfinal.pdf). + +SFTP isn't supported under plan9 until [this +issue](https://github.com/pkg/sftp/issues/156) is fixed. + +Note that since SFTP isn't HTTP based the following flags don't work +with it: `--dump-headers`, `--dump-bodies`, `--dump-auth` + +Note that `--timeout` isn't supported (but `--contimeout` is). + +WebDAV +----------------------------------------- + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +To configure the WebDAV remote you will need to have a URL for it, and +a username and password. If you know what kind of system you are +connecting to then rclone can enable extra features. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value +[snip] +22 / Webdav + \ "webdav" +[snip] +Storage> webdav +URL of http host to connect to +Choose a number from below, or type in your own value + 1 / Connect to example.com + \ "https://example.com" +url> https://example.com/remote.php/webdav/ +Name of the Webdav site/service/software you are using +Choose a number from below, or type in your own value + 1 / Nextcloud + \ "nextcloud" + 2 / Owncloud + \ "owncloud" + 3 / Sharepoint + \ "sharepoint" + 4 / Other site/service or software + \ "other" +vendor> 1 +User name +user> user +Password. +y) Yes type in my own password +g) Generate random password +n) No leave this optional password blank +y/g/n> y +Enter the password: +password: +Confirm the password: +password: +Bearer token instead of user/pass (eg a Macaroon) +bearer_token> +Remote config +-------------------- +[remote] +type = webdav +url = https://example.com/remote.php/webdav/ +vendor = nextcloud +user = user +pass = *** ENCRYPTED *** +bearer_token = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Once configured you can then use `rclone` like this, + +List directories in top level of your WebDAV + + rclone lsd remote: + +List all the files in your WebDAV + + rclone ls remote: + +To copy a local directory to an WebDAV directory called backup + + rclone copy /home/source remote:backup + +### Modified time and hashes ### + +Plain WebDAV does not support modified times. However when used with +Owncloud or Nextcloud rclone will support modified times. + +Hashes are not supported. + +## Provider notes ## + +See below for notes on specific providers. + +### Owncloud ### + +Click on the settings cog in the bottom right of the page and this +will show the WebDAV URL that rclone needs in the config step. It +will look something like `https://example.com/remote.php/webdav/`. + +Owncloud supports modified times using the `X-OC-Mtime` header. + +### Nextcloud ### + +This is configured in an identical way to Owncloud. Note that +Nextcloud does not support streaming of files (`rcat`) whereas +Owncloud does. This [may be +fixed](https://github.com/nextcloud/nextcloud-snap/issues/365) in the +future. + +### Put.io ### + +put.io can be accessed in a read only way using webdav. + +Configure the `url` as `https://webdav.put.io` and use your normal +account username and password for `user` and `pass`. Set the `vendor` +to `other`. + +Your config file should end up looking like this: + +``` +[putio] +type = webdav +url = https://webdav.put.io +vendor = other +user = YourUserName +pass = encryptedpassword +``` + +If you are using `put.io` with `rclone mount` then use the +`--read-only` flag to signal to the OS that it can't write to the +mount. + +For more help see [the put.io webdav docs](http://help.put.io/apps-and-integrations/ftp-and-webdav). + +### Sharepoint ### + +Rclone can be used with Sharepoint provided by OneDrive for Business +or Office365 Education Accounts. +This feature is only needed for a few of these Accounts, +mostly Office365 Education ones. These accounts are sometimes not +verified by the domain owner [github#1975](https://github.com/ncw/rclone/issues/1975) + +This means that these accounts can't be added using the official +API (other Accounts should work with the "onedrive" option). However, +it is possible to access them using webdav. + +To use a sharepoint remote with rclone, add it like this: +First, you need to get your remote's URL: + +- Go [here](https://onedrive.live.com/about/en-us/signin/) + to open your OneDrive or to sign in +- Now take a look at your address bar, the URL should look like this: + `https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/_layouts/15/onedrive.aspx` + +You'll only need this URL upto the email address. After that, you'll +most likely want to add "/Documents". That subdirectory contains +the actual data stored on your OneDrive. + +Add the remote to rclone like this: +Configure the `url` as `https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/Documents` +and use your normal account email and password for `user` and `pass`. +If you have 2FA enabled, you have to generate an app password. +Set the `vendor` to `sharepoint`. + +Your config file should look like this: + +``` +[sharepoint] +type = webdav +url = https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/Documents +vendor = other +user = YourEmailAddress +pass = encryptedpassword +``` + +### dCache ### + +dCache is a storage system with WebDAV doors that support, beside basic and x509, +authentication with [Macaroons](https://www.dcache.org/manuals/workshop-2017-05-29-Umea/000-Final/anupam_macaroons_v02.pdf) (bearer tokens). + +Configure as normal using the `other` type. Don't enter a username or +password, instead enter your Macaroon as the `bearer_token`. + +The config will end up looking something like this. + +``` +[dcache] +type = webdav +url = https://dcache... +vendor = other +user = +pass = +bearer_token = your-macaroon +``` + +There is a [script](https://github.com/onnozweers/dcache-scripts/blob/master/get-share-link) that +obtains a Macaroon from a dCache WebDAV endpoint, and creates an rclone config file. + +Yandex Disk +---------------------------------------- + +[Yandex Disk](https://disk.yandex.com) is a cloud storage solution created by [Yandex](https://yandex.com). + +Yandex paths may be as deep as required, eg `remote:directory/subdirectory`. + +Here is an example of making a yandex configuration. First run + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +n/s> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" +10 / Microsoft OneDrive + \ "onedrive" +11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +12 / SSH/SFTP Connection + \ "sftp" +13 / Yandex Disk + \ "yandex" +Storage> 13 +Yandex Client Id - leave blank normally. +client_id> +Yandex Client Secret - leave blank normally. +client_secret> +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +client_id = +client_secret = +token = {"access_token":"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","token_type":"bearer","expiry":"2016-12-29T12:27:11.362788025Z"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Yandex Disk. This only runs from the moment it +opens your browser to the moment you get back the verification code. +This is on `http://127.0.0.1:53682/` and this it may require you to +unblock it temporarily if you are running a host firewall. + +Once configured you can then use `rclone` like this, + +See top level directories + + rclone lsd remote: + +Make a new directory + + rclone mkdir remote:directory + +List the contents of a directory + + rclone ls remote:directory + +Sync `/home/local/directory` to the remote path, deleting any +excess files in the path. + + rclone sync /home/local/directory remote:directory + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +### Modified time ### + +Modified times are supported and are stored accurate to 1 ns in custom +metadata called `rclone_modified` in RFC3339 with nanoseconds format. + +### MD5 checksums ### + +MD5 checksums are natively supported by Yandex Disk. + +### Emptying Trash ### + +If you wish to empty your trash you can use the `rclone cleanup remote:` +command which will permanently delete all your trashed files. This command +does not take any path arguments. + +Local Filesystem +------------------------------------------- + +Local paths are specified as normal filesystem paths, eg `/path/to/wherever`, so + + rclone sync /home/source /tmp/destination + +Will sync `/home/source` to `/tmp/destination` + +These can be configured into the config file for consistencies sake, +but it is probably easier not to. + +### Modified time ### + +Rclone reads and writes the modified time using an accuracy determined by +the OS. Typically this is 1ns on Linux, 10 ns on Windows and 1 Second +on OS X. + +### Filenames ### + +Filenames are expected to be encoded in UTF-8 on disk. This is the +normal case for Windows and OS X. + +There is a bit more uncertainty in the Linux world, but new +distributions will have UTF-8 encoded files names. If you are using an +old Linux filesystem with non UTF-8 file names (eg latin1) then you +can use the `convmv` tool to convert the filesystem to UTF-8. This +tool is available in most distributions' package managers. + +If an invalid (non-UTF8) filename is read, the invalid characters will +be replaced with the unicode replacement character, '�'. `rclone` +will emit a debug message in this case (use `-v` to see), eg + +``` +Local file system at .: Replacing invalid UTF-8 characters in "gro\xdf" +``` + +### Long paths on Windows ### + +Rclone handles long paths automatically, by converting all paths to long +[UNC paths](https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath) +which allows paths up to 32,767 characters. + +This is why you will see that your paths, for instance `c:\files` is +converted to the UNC path `\\?\c:\files` in the output, +and `\\server\share` is converted to `\\?\UNC\server\share`. + +However, in rare cases this may cause problems with buggy file +system drivers like [EncFS](https://github.com/ncw/rclone/issues/261). +To disable UNC conversion globally, add this to your `.rclone.conf` file: + +``` +[local] +nounc = true +``` + +If you want to selectively disable UNC, you can add it to a separate entry like this: + +``` +[nounc] +type = local +nounc = true +``` +And use rclone like this: + +`rclone copy c:\src nounc:z:\dst` + +This will use UNC paths on `c:\src` but not on `z:\dst`. +Of course this will cause problems if the absolute path length of a +file exceeds 258 characters on z, so only use this option if you have to. + +### Specific options ### + +Here are the command line options specific to local storage + +#### --copy-links, -L #### + +Normally rclone will ignore symlinks or junction points (which behave +like symlinks under Windows). + +If you supply this flag then rclone will follow the symlink and copy +the pointed to file or directory. + +This flag applies to all commands. + +For example, supposing you have a directory structure like this + +``` +$ tree /tmp/a +/tmp/a +├── b -> ../b +├── expected -> ../expected +├── one +└── two + └── three +``` + +Then you can see the difference with and without the flag like this + +``` +$ rclone ls /tmp/a + 6 one + 6 two/three +``` + +and + +``` +$ rclone -L ls /tmp/a + 4174 expected + 6 one + 6 two/three + 6 b/two + 6 b/one +``` + +#### --local-no-check-updated #### + +Don't check to see if the files change during upload. + +Normally rclone checks the size and modification time of files as they +are being uploaded and aborts with a message which starts `can't copy +- source file is being updated` if the file changes during upload. + +However on some file systems this modification time check may fail (eg +[Glusterfs #2206](https://github.com/ncw/rclone/issues/2206)) so this +check can be disabled with this flag. + +#### --local-no-unicode-normalization #### + +This flag is deprecated now. Rclone no longer normalizes unicode file +names, but it compares them with unicode normalization in the sync +routine instead. + +#### --one-file-system, -x #### + +This tells rclone to stay in the filesystem specified by the root and +not to recurse into different file systems. + +For example if you have a directory hierarchy like this + +``` +root +├── disk1 - disk1 mounted on the root +│   └── file3 - stored on disk1 +├── disk2 - disk2 mounted on the root +│   └── file4 - stored on disk12 +├── file1 - stored on the root disk +└── file2 - stored on the root disk +``` + +Using `rclone --one-file-system copy root remote:` will only copy `file1` and `file2`. Eg + +``` +$ rclone -q --one-file-system ls root + 0 file1 + 0 file2 +``` + +``` +$ rclone -q ls root + 0 disk1/file3 + 0 disk2/file4 + 0 file1 + 0 file2 +``` + +**NB** Rclone (like most unix tools such as `du`, `rsync` and `tar`) +treats a bind mount to the same device as being on the same +filesystem. + +**NB** This flag is only available on Unix based systems. On systems +where it isn't supported (eg Windows) it will not appear as an valid +flag. + +#### --skip-links #### + +This flag disables warning messages on skipped symlinks or junction +points, as you explicitly acknowledge that they should be skipped. + +# Changelog + +## v1.43.1 - 2018-09-07 + +Point release to fix hubic and azureblob backends. + +* Bug Fixes + * ncdu: Return error instead of log.Fatal in Show (Fabian Möller) + * cmd: Fix crash with --progress and --stats 0 (Nick Craig-Wood) + * docs: Tidy website display (Anagh Kumar Baranwal) +* Azure Blob: + * Fix multi-part uploads. (sandeepkru) +* Hubic + * Fix uploads (Nick Craig-Wood) + * Retry auth fetching if it fails to make hubic more reliable (Nick Craig-Wood) + +## v1.43 - 2018-09-01 + +* New backends + * Jottacloud (Sebastian Bünger) +* New commands + * copyurl: copies a URL to a remote (Denis) +* New Features + * Reworked config for backends (Nick Craig-Wood) + * All backend config can now be supplied by command line, env var or config file + * Advanced section in the config wizard for the optional items + * A large step towards rclone backends being usable in other go software + * Allow on the fly remotes with :backend: syntax + * Stats revamp + * Add `--progress`/`-P` flag to show interactive progress (Nick Craig-Wood) + * Show the total progress of the sync in the stats (Nick Craig-Wood) + * Add `--stats-one-line` flag for single line stats (Nick Craig-Wood) + * Added weekday schedule into `--bwlimit` (Mateusz) + * lsjson: Add option to show the original object IDs (Fabian Möller) + * serve webdav: Make Content-Type without reading the file and add `--etag-hash` (Nick Craig-Wood) + * build + * Build macOS with native compiler (Nick Craig-Wood) + * Update to use go1.11 for the build (Nick Craig-Wood) + * rc + * Added core/stats to return the stats (reddi1) + * `version --check`: Prints the current release and beta versions (Nick Craig-Wood) +* Bug Fixes + * accounting + * Fix time to completion estimates (Nick Craig-Wood) + * Fix moving average speed for file stats (Nick Craig-Wood) + * config: Fix error reading password from piped input (Nick Craig-Wood) + * move: Fix `--delete-empty-src-dirs` flag to delete all empty dirs on move (ishuah) +* Mount + * Implement `--daemon-timeout` flag for OSXFUSE (Nick Craig-Wood) + * Fix mount `--daemon` not working with encrypted config (Alex Chen) + * Clip the number of blocks to 2^32-1 on macOS - fixes borg backup (Nick Craig-Wood) +* VFS + * Enable vfs-read-chunk-size by default (Fabian Möller) + * Add the vfs/refresh rc command (Fabian Möller) + * Add non recursive mode to vfs/refresh rc command (Fabian Möller) + * Try to seek buffer on read only files (Fabian Möller) +* Local + * Fix crash when deprecated `--local-no-unicode-normalization` is supplied (Nick Craig-Wood) + * Fix mkdir error when trying to copy files to the root of a drive on windows (Nick Craig-Wood) +* Cache + * Fix nil pointer deref when using lsjson on cached directory (Nick Craig-Wood) + * Fix nil pointer deref for occasional crash on playback (Nick Craig-Wood) +* Crypt + * Fix accounting when checking hashes on upload (Nick Craig-Wood) +* Amazon Cloud Drive + * Make very clear in the docs that rclone has no ACD keys (Nick Craig-Wood) +* Azure Blob + * Add connection string and SAS URL auth (Nick Craig-Wood) + * List the container to see if it exists (Nick Craig-Wood) + * Port new Azure Blob Storage SDK (sandeepkru) + * Added blob tier, tier between Hot, Cool and Archive. (sandeepkru) + * Remove leading / from paths (Nick Craig-Wood) +* B2 + * Support Application Keys (Nick Craig-Wood) + * Remove leading / from paths (Nick Craig-Wood) +* Box + * Fix upload of > 2GB files on 32 bit platforms (Nick Craig-Wood) + * Make `--box-commit-retries` flag defaulting to 100 to fix large uploads (Nick Craig-Wood) +* Drive + * Add `--drive-keep-revision-forever` flag (lewapm) + * Handle gdocs when filtering file names in list (Fabian Möller) + * Support using `--fast-list` for large speedups (Fabian Möller) +* FTP + * Fix Put mkParentDir failed: 521 for BunnyCDN (Nick Craig-Wood) +* Google Cloud Storage + * Fix index out of range error with `--fast-list` (Nick Craig-Wood) +* Jottacloud + * Fix MD5 error check (Oliver Heyme) + * Handle empty time values (Martin Polden) + * Calculate missing MD5s (Oliver Heyme) + * Docs, fixes and tests for MD5 calculation (Nick Craig-Wood) + * Add optional MimeTyper interface. (Sebastian Bünger) + * Implement optional About interface (for `df` support). (Sebastian Bünger) +* Mega + * Wait for events instead of arbitrary sleeping (Nick Craig-Wood) + * Add `--mega-hard-delete` flag (Nick Craig-Wood) + * Fix failed logins with upper case chars in email (Nick Craig-Wood) +* Onedrive + * Shared folder support (Yoni Jah) + * Implement DirMove (Cnly) + * Fix rmdir sometimes deleting directories with contents (Nick Craig-Wood) +* Pcloud + * Delete half uploaded files on upload error (Nick Craig-Wood) +* Qingstor + * Remove leading / from paths (Nick Craig-Wood) +* S3 + * Fix index out of range error with `--fast-list` (Nick Craig-Wood) + * Add `--s3-force-path-style` (Nick Craig-Wood) + * Add support for KMS Key ID (bsteiss) + * Remove leading / from paths (Nick Craig-Wood) +* Swift + * Add `storage_policy` (Ruben Vandamme) + * Make it so just `storage_url` or `auth_token` can be overidden (Nick Craig-Wood) + * Fix server side copy bug for unusal file names (Nick Craig-Wood) + * Remove leading / from paths (Nick Craig-Wood) +* WebDAV + * Ensure we call MKCOL with a URL with a trailing / for QNAP interop (Nick Craig-Wood) + * If root ends with / then don't check if it is a file (Nick Craig-Wood) + * Don't accept redirects when reading metadata (Nick Craig-Wood) + * Add bearer token (Macaroon) support for dCache (Nick Craig-Wood) + * Document dCache and Macaroons (Onno Zweers) + * Sharepoint recursion with different depth (Henning) + * Attempt to remove failed uploads (Nick Craig-Wood) +* Yandex + * Fix listing/deleting files in the root (Nick Craig-Wood) + +## v1.42 - 2018-06-16 + +* New backends + * OpenDrive (Oliver Heyme, Jakub Karlicek, ncw) +* New commands + * deletefile command (Filip Bartodziej) +* New Features + * copy, move: Copy single files directly, don't use `--files-from` work-around + * this makes them much more efficient + * Implement `--max-transfer` flag to quit transferring at a limit + * make exit code 8 for `--max-transfer` exceeded + * copy: copy empty source directories to destination (Ishuah Kariuki) + * check: Add `--one-way` flag (Kasper Byrdal Nielsen) + * Add siginfo handler for macOS for ctrl-T stats (kubatasiemski) + * rc + * add core/gc to run a garbage collection on demand + * enable go profiling by default on the `--rc` port + * return error from remote on failure + * lsf + * Add `--absolute` flag to add a leading / onto path names + * Add `--csv` flag for compliant CSV output + * Add 'm' format specifier to show the MimeType + * Implement 'i' format for showing object ID + * lsjson + * Add MimeType to the output + * Add ID field to output to show Object ID + * Add `--retries-sleep` flag (Benjamin Joseph Dag) + * Oauth tidy up web page and error handling (Henning Surmeier) +* Bug Fixes + * Password prompt output with `--log-file` fixed for unix (Filip Bartodziej) + * Calculate ModifyWindow each time on the fly to fix various problems (Stefan Breunig) +* Mount + * Only print "File.rename error" if there actually is an error (Stefan Breunig) + * Delay rename if file has open writers instead of failing outright (Stefan Breunig) + * Ensure atexit gets run on interrupt + * macOS enhancements + * Make `--noappledouble` `--noapplexattr` + * Add `--volname` flag and remove special chars from it + * Make Get/List/Set/Remove xattr return ENOSYS for efficiency + * Make `--daemon` work for macOS without CGO +* VFS + * Add `--vfs-read-chunk-size` and `--vfs-read-chunk-size-limit` (Fabian Möller) + * Fix ChangeNotify for new or changed folders (Fabian Möller) +* Local + * Fix symlink/junction point directory handling under Windows + * **NB** you will need to add `-L` to your command line to copy files with reparse points +* Cache + * Add non cached dirs on notifications (Remus Bunduc) + * Allow root to be expired from rc (Remus Bunduc) + * Clean remaining empty folders from temp upload path (Remus Bunduc) + * Cache lists using batch writes (Remus Bunduc) + * Use secure websockets for HTTPS Plex addresses (John Clayton) + * Reconnect plex websocket on failures (Remus Bunduc) + * Fix panic when running without plex configs (Remus Bunduc) + * Fix root folder caching (Remus Bunduc) +* Crypt + * Check the crypted hash of files when uploading for extra data security +* Dropbox + * Make Dropbox for business folders accessible using an initial `/` in the path +* Google Cloud Storage + * Low level retry all operations if necessary +* Google Drive + * Add `--drive-acknowledge-abuse` to download flagged files + * Add `--drive-alternate-export` to fix large doc export + * Don't attempt to choose Team Drives when using rclone config create + * Fix change list polling with team drives + * Fix ChangeNotify for folders (Fabian Möller) + * Fix about (and df on a mount) for team drives +* Onedrive + * Errorhandler for onedrive for business requests (Henning Surmeier) +* S3 + * Adjust upload concurrency with `--s3-upload-concurrency` (themylogin) + * Fix `--s3-chunk-size` which was always using the minimum +* SFTP + * Add `--ssh-path-override` flag (Piotr Oleszczyk) + * Fix slow downloads for long latency connections +* Webdav + * Add workarounds for biz.mail.ru + * Ignore Reason-Phrase in status line to fix 4shared (Rodrigo) + * Better error message generation + +## v1.41 - 2018-04-28 + +* New backends + * Mega support added + * Webdav now supports SharePoint cookie authentication (hensur) +* New commands + * link: create public link to files and folders (Stefan Breunig) + * about: gets quota info from a remote (a-roussos, ncw) + * hashsum: a generic tool for any hash to produce md5sum like output +* New Features + * lsd: Add -R flag and fix and update docs for all ls commands + * ncdu: added a "refresh" key - CTRL-L (Keith Goldfarb) + * serve restic: Add append-only mode (Steve Kriss) + * serve restic: Disallow overwriting files in append-only mode (Alexander Neumann) + * serve restic: Print actual listener address (Matt Holt) + * size: Add --json flag (Matthew Holt) + * sync: implement --ignore-errors (Mateusz Pabian) + * dedupe: Add dedupe largest functionality (Richard Yang) + * fs: Extend SizeSuffix to include TB and PB for rclone about + * fs: add --dump goroutines and --dump openfiles for debugging + * rc: implement core/memstats to print internal memory usage info + * rc: new call rc/pid (Michael P. Dubner) +* Compile + * Drop support for go1.6 +* Release + * Fix `make tarball` (Chih-Hsuan Yen) +* Bug Fixes + * filter: fix --min-age and --max-age together check + * fs: limit MaxIdleConns and MaxIdleConnsPerHost in transport + * lsd,lsf: make sure all times we output are in local time + * rc: fix setting bwlimit to unlimited + * rc: take note of the --rc-addr flag too as per the docs +* Mount + * Use About to return the correct disk total/used/free (eg in `df`) + * Set `--attr-timeout default` to `1s` - fixes: + * rclone using too much memory + * rclone not serving files to samba + * excessive time listing directories + * Fix `df -i` (upstream fix) +* VFS + * Filter files `.` and `..` from directory listing + * Only make the VFS cache if --vfs-cache-mode > Off +* Local + * Add --local-no-check-updated to disable updated file checks + * Retry remove on Windows sharing violation error +* Cache + * Flush the memory cache after close + * Purge file data on notification + * Always forget parent dir for notifications + * Integrate with Plex websocket + * Add rc cache/stats (seuffert) + * Add info log on notification +* Box + * Fix failure reading large directories - parse file/directory size as float +* Dropbox + * Fix crypt+obfuscate on dropbox + * Fix repeatedly uploading the same files +* FTP + * Work around strange response from box FTP server + * More workarounds for FTP servers to fix mkParentDir error + * Fix no error on listing non-existent directory +* Google Cloud Storage + * Add service_account_credentials (Matt Holt) + * Detect bucket presence by listing it - minimises permissions needed + * Ignore zero length directory markers +* Google Drive + * Add service_account_credentials (Matt Holt) + * Fix directory move leaving a hardlinked directory behind + * Return proper google errors when Opening files + * When initialized with a filepath, optional features used incorrect root path (Stefan Breunig) +* HTTP + * Fix sync for servers which don't return Content-Length in HEAD +* Onedrive + * Add QuickXorHash support for OneDrive for business + * Fix socket leak in multipart session upload +* S3 + * Look in S3 named profile files for credentials + * Add `--s3-disable-checksum` to disable checksum uploading (Chris Redekop) + * Hierarchical configuration support (Giri Badanahatti) + * Add in config for all the supported S3 providers + * Add One Zone Infrequent Access storage class (Craig Rachel) + * Add --use-server-modtime support (Peter Baumgartner) + * Add --s3-chunk-size option to control multipart uploads + * Ignore zero length directory markers +* SFTP + * Update docs to match code, fix typos and clarify disable_hashcheck prompt (Michael G. Noll) + * Update docs with Synology quirks + * Fail soft with a debug on hash failure +* Swift + * Add --use-server-modtime support (Peter Baumgartner) +* Webdav + * Support SharePoint cookie authentication (hensur) + * Strip leading and trailing / off root + +## v1.40 - 2018-03-19 + +* New backends + * Alias backend to create aliases for existing remote names (Fabian Möller) +* New commands + * `lsf`: list for parsing purposes (Jakub Tasiemski) + * by default this is a simple non recursive list of files and directories + * it can be configured to add more info in an easy to parse way + * `serve restic`: for serving a remote as a Restic REST endpoint + * This enables restic to use any backends that rclone can access + * Thanks Alexander Neumann for help, patches and review + * `rc`: enable the remote control of a running rclone + * The running rclone must be started with --rc and related flags. + * Currently there is support for bwlimit, and flushing for mount and cache. +* New Features + * `--max-delete` flag to add a delete threshold (Bjørn Erik Pedersen) + * All backends now support RangeOption for ranged Open + * `cat`: Use RangeOption for limited fetches to make more efficient + * `cryptcheck`: make reading of nonce more efficient with RangeOption + * serve http/webdav/restic + * support SSL/TLS + * add `--user` `--pass` and `--htpasswd` for authentication + * `copy`/`move`: detect file size change during copy/move and abort transfer (ishuah) + * `cryptdecode`: added option to return encrypted file names. (ishuah) + * `lsjson`: add `--encrypted` to show encrypted name (Jakub Tasiemski) + * Add `--stats-file-name-length` to specify the printed file name length for stats (Will Gunn) +* Compile + * Code base was shuffled and factored + * backends moved into a backend directory + * large packages split up + * See the CONTRIBUTING.md doc for info as to what lives where now + * Update to using go1.10 as the default go version + * Implement daily [full integration tests](https://pub.rclone.org/integration-tests/) +* Release + * Include a source tarball and sign it and the binaries + * Sign the git tags as part of the release process + * Add .deb and .rpm packages as part of the build + * Make a beta release for all branches on the main repo (but not pull requests) +* Bug Fixes + * config: fixes errors on non existing config by loading config file only on first access + * config: retry saving the config after failure (Mateusz) + * sync: when using `--backup-dir` don't delete files if we can't set their modtime + * this fixes odd behaviour with Dropbox and `--backup-dir` + * fshttp: fix idle timeouts for HTTP connections + * `serve http`: fix serving files with : in - fixes + * Fix `--exclude-if-present` to ignore directories which it doesn't have permission for (Iakov Davydov) + * Make accounting work properly with crypt and b2 + * remove `--no-traverse` flag because it is obsolete +* Mount + * Add `--attr-timeout` flag to control attribute caching in kernel + * this now defaults to 0 which is correct but less efficient + * see [the mount docs](/commands/rclone_mount/#attribute-caching) for more info + * Add `--daemon` flag to allow mount to run in the background (ishuah) + * Fix: Return ENOSYS rather than EIO on attempted link + * This fixes FileZilla accessing an rclone mount served over sftp. + * Fix setting modtime twice + * Mount tests now run on CI for Linux (mount & cmount)/Mac/Windows + * Many bugs fixed in the VFS layer - see below +* VFS + * Many fixes for `--vfs-cache-mode` writes and above + * Update cached copy if we know it has changed (fixes stale data) + * Clean path names before using them in the cache + * Disable cache cleaner if `--vfs-cache-poll-interval=0` + * Fill and clean the cache immediately on startup + * Fix Windows opening every file when it stats the file + * Fix applying modtime for an open Write Handle + * Fix creation of files when truncating + * Write 0 bytes when flushing unwritten handles to avoid race conditions in FUSE + * Downgrade "poll-interval is not supported" message to Info + * Make OpenFile and friends return EINVAL if O_RDONLY and O_TRUNC +* Local + * Downgrade "invalid cross-device link: trying copy" to debug + * Make DirMove return fs.ErrorCantDirMove to allow fallback to Copy for cross device + * Fix race conditions updating the hashes +* Cache + * Add support for polling - cache will update when remote changes on supported backends + * Reduce log level for Plex api + * Fix dir cache issue + * Implement `--cache-db-wait-time` flag + * Improve efficiency with RangeOption and RangeSeek + * Fix dirmove with temp fs enabled + * Notify vfs when using temp fs + * Offline uploading + * Remote control support for path flushing +* Amazon cloud drive + * Rclone no longer has any working keys - disable integration tests + * Implement DirChangeNotify to notify cache/vfs/mount of changes +* Azureblob + * Don't check for bucket/container presense if listing was OK + * this makes rclone do one less request per invocation + * Improve accounting for chunked uploads +* Backblaze B2 + * Don't check for bucket/container presense if listing was OK + * this makes rclone do one less request per invocation +* Box + * Improve accounting for chunked uploads +* Dropbox + * Fix custom oauth client parameters +* Google Cloud Storage + * Don't check for bucket/container presense if listing was OK + * this makes rclone do one less request per invocation +* Google Drive + * Migrate to api v3 (Fabian Möller) + * Add scope configuration and root folder selection + * Add `--drive-impersonate` for service accounts + * thanks to everyone who tested, explored and contributed docs + * Add `--drive-use-created-date` to use created date as modified date (nbuchanan) + * Request the export formats only when required + * This makes rclone quicker when there are no google docs + * Fix finding paths with latin1 chars (a workaround for a drive bug) + * Fix copying of a single Google doc file + * Fix `--drive-auth-owner-only` to look in all directories +* HTTP + * Fix handling of directories with & in +* Onedrive + * Removed upload cutoff and always do session uploads + * this stops the creation of multiple versions on business onedrive + * Overwrite object size value with real size when reading file. (Victor) + * this fixes oddities when onedrive misreports the size of images +* Pcloud + * Remove unused chunked upload flag and code +* Qingstor + * Don't check for bucket/container presense if listing was OK + * this makes rclone do one less request per invocation +* S3 + * Support hashes for multipart files (Chris Redekop) + * Initial support for IBM COS (S3) (Giri Badanahatti) + * Update docs to discourage use of v2 auth with CEPH and others + * Don't check for bucket/container presense if listing was OK + * this makes rclone do one less request per invocation + * Fix server side copy and set modtime on files with + in +* SFTP + * Add option to disable remote hash check command execution (Jon Fautley) + * Add `--sftp-ask-password` flag to prompt for password when needed (Leo R. Lundgren) + * Add `set_modtime` configuration option + * Fix following of symlinks + * Fix reading config file outside of Fs setup + * Fix reading $USER in username fallback not $HOME + * Fix running under crontab - Use correct OS way of reading username +* Swift + * Fix refresh of authentication token + * in v1.39 a bug was introduced which ignored new tokens - this fixes it + * Fix extra HEAD transaction when uploading a new file + * Don't check for bucket/container presense if listing was OK + * this makes rclone do one less request per invocation +* Webdav + * Add new time formats to support mydrive.ch and others + +## v1.39 - 2017-12-23 + +* New backends + * WebDAV + * tested with nextcloud, owncloud, put.io and others! + * Pcloud + * cache - wraps a cache around other backends (Remus Bunduc) + * useful in combination with mount + * NB this feature is in beta so use with care +* New commands + * serve command with subcommands: + * serve webdav: this implements a webdav server for any rclone remote. + * serve http: command to serve a remote over HTTP + * config: add sub commands for full config file management + * create/delete/dump/edit/file/password/providers/show/update + * touch: to create or update the timestamp of a file (Jakub Tasiemski) +* New Features + * curl install for rclone (Filip Bartodziej) + * --stats now shows percentage, size, rate and ETA in condensed form (Ishuah Kariuki) + * --exclude-if-present to exclude a directory if a file is present (Iakov Davydov) + * rmdirs: add --leave-root flag (lewpam) + * move: add --delete-empty-src-dirs flag to remove dirs after move (Ishuah Kariuki) + * Add --dump flag, introduce --dump requests, responses and remove --dump-auth, --dump-filters + * Obscure X-Auth-Token: from headers when dumping too + * Document and implement exit codes for different failure modes (Ishuah Kariuki) +* Compile +* Bug Fixes + * Retry lots more different types of errors to make multipart transfers more reliable + * Save the config before asking for a token, fixes disappearing oauth config + * Warn the user if --include and --exclude are used together (Ernest Borowski) + * Fix duplicate files (eg on Google drive) causing spurious copies + * Allow trailing and leading whitespace for passwords (Jason Rose) + * ncdu: fix crashes on empty directories + * rcat: fix goroutine leak + * moveto/copyto: Fix to allow copying to the same name +* Mount + * --vfs-cache mode to make writes into mounts more reliable. + * this requires caching files on the disk (see --cache-dir) + * As this is a new feature, use with care + * Use sdnotify to signal systemd the mount is ready (Fabian Möller) + * Check if directory is not empty before mounting (Ernest Borowski) +* Local + * Add error message for cross file system moves + * Fix equality check for times +* Dropbox + * Rework multipart upload + * buffer the chunks when uploading large files so they can be retried + * change default chunk size to 48MB now we are buffering them in memory + * retry every error after the first chunk is done successfully + * Fix error when renaming directories +* Swift + * Fix crash on bad authentication +* Google Drive + * Add service account support (Tim Cooijmans) +* S3 + * Make it work properly with Digital Ocean Spaces (Andrew Starr-Bochicchio) + * Fix crash if a bad listing is received + * Add support for ECS task IAM roles (David Minor) +* Backblaze B2 + * Fix multipart upload retries + * Fix --hard-delete to make it work 100% of the time +* Swift + * Allow authentication with storage URL and auth key (Giovanni Pizzi) + * Add new fields for swift configuration to support IBM Bluemix Swift (Pierre Carlson) + * Add OS_TENANT_ID and OS_USER_ID to config + * Allow configs with user id instead of user name + * Check if swift segments container exists before creating (John Leach) + * Fix memory leak in swift transfers (upstream fix) +* SFTP + * Add option to enable the use of aes128-cbc cipher (Jon Fautley) +* Amazon cloud drive + * Fix download of large files failing with "Only one auth mechanism allowed" +* crypt + * Option to encrypt directory names or leave them intact + * Implement DirChangeNotify (Fabian Möller) +* onedrive + * Add option to choose resourceURL during setup of OneDrive Business account if more than one is available for user + +## v1.38 - 2017-09-30 + +* New backends + * Azure Blob Storage (thanks Andrei Dragomir) + * Box + * Onedrive for Business (thanks Oliver Heyme) + * QingStor from QingCloud (thanks wuyu) +* New commands + * `rcat` - read from standard input and stream upload + * `tree` - shows a nicely formatted recursive listing + * `cryptdecode` - decode crypted file names (thanks ishuah) + * `config show` - print the config file + * `config file` - print the config file location +* New Features + * Empty directories are deleted on `sync` + * `dedupe` - implement merging of duplicate directories + * `check` and `cryptcheck` made more consistent and use less memory + * `cleanup` for remaining remotes (thanks ishuah) + * `--immutable` for ensuring that files don't change (thanks Jacob McNamee) + * `--user-agent` option (thanks Alex McGrath Kraak) + * `--disable` flag to disable optional features + * `--bind` flag for choosing the local addr on outgoing connections + * Support for zsh auto-completion (thanks bpicode) + * Stop normalizing file names but do a normalized compare in `sync` +* Compile + * Update to using go1.9 as the default go version + * Remove snapd build due to maintenance problems +* Bug Fixes + * Improve retriable error detection which makes multipart uploads better + * Make `check` obey `--ignore-size` + * Fix bwlimit toggle in conjunction with schedules (thanks cbruegg) + * `config` ensures newly written config is on the same mount +* Local + * Revert to copy when moving file across file system boundaries + * `--skip-links` to suppress symlink warnings (thanks Zhiming Wang) +* Mount + * Re-use `rcat` internals to support uploads from all remotes +* Dropbox + * Fix "entry doesn't belong in directory" error + * Stop using deprecated API methods +* Swift + * Fix server side copy to empty container with `--fast-list` +* Google Drive + * Change the default for `--drive-use-trash` to `true` +* S3 + * Set session token when using STS (thanks Girish Ramakrishnan) + * Glacier docs and error messages (thanks Jan Varho) + * Read 1000 (not 1024) items in dir listings to fix Wasabi +* Backblaze B2 + * Fix SHA1 mismatch when downloading files with no SHA1 + * Calculate missing hashes on the fly instead of spooling + * `--b2-hard-delete` to permanently delete (not hide) files (thanks John Papandriopoulos) +* Hubic + * Fix creating containers - no longer have to use the `default` container +* Swift + * Optionally configure from a standard set of OpenStack environment vars + * Add `endpoint_type` config +* Google Cloud Storage + * Fix bucket creation to work with limited permission users +* SFTP + * Implement connection pooling for multiple ssh connections + * Limit new connections per second + * Add support for MD5 and SHA1 hashes where available (thanks Christian Brüggemann) +* HTTP + * Fix URL encoding issues + * Fix directories with `:` in + * Fix panic with URL encoded content + +## v1.37 - 2017-07-22 + +* New backends + * FTP - thanks to Antonio Messina + * HTTP - thanks to Vasiliy Tolstov +* New commands + * rclone ncdu - for exploring a remote with a text based user interface. + * rclone lsjson - for listing with a machine readable output + * rclone dbhashsum - to show Dropbox style hashes of files (local or Dropbox) +* New Features + * Implement --fast-list flag + * This allows remotes to list recursively if they can + * This uses less transactions (important if you pay for them) + * This may or may not be quicker + * This will use more memory as it has to hold the listing in memory + * --old-sync-method deprecated - the remaining uses are covered by --fast-list + * This involved a major re-write of all the listing code + * Add --tpslimit and --tpslimit-burst to limit transactions per second + * this is useful in conjuction with `rclone mount` to limit external apps + * Add --stats-log-level so can see --stats without -v + * Print password prompts to stderr - Hraban Luyat + * Warn about duplicate files when syncing + * Oauth improvements + * allow auth_url and token_url to be set in the config file + * Print redirection URI if using own credentials. + * Don't Mkdir at the start of sync to save transactions +* Compile + * Update build to go1.8.3 + * Require go1.6 for building rclone + * Compile 386 builds with "GO386=387" for maximum compatibility +* Bug Fixes + * Fix menu selection when no remotes + * Config saving reworked to not kill the file if disk gets full + * Don't delete remote if name does not change while renaming + * moveto, copyto: report transfers and checks as per move and copy +* Local + * Add --local-no-unicode-normalization flag - Bob Potter +* Mount + * Now supported on Windows using cgofuse and WinFsp - thanks to Bill Zissimopoulos for much help + * Compare checksums on upload/download via FUSE + * Unmount when program ends with SIGINT (Ctrl+C) or SIGTERM - Jérôme Vizcaino + * On read only open of file, make open pending until first read + * Make --read-only reject modify operations + * Implement ModTime via FUSE for remotes that support it + * Allow modTime to be changed even before all writers are closed + * Fix panic on renames + * Fix hang on errored upload +* Crypt + * Report the name:root as specified by the user + * Add an "obfuscate" option for filename encryption - Stephen Harris +* Amazon Drive + * Fix initialization order for token renewer + * Remove revoked credentials, allow oauth proxy config and update docs +* B2 + * Reduce minimum chunk size to 5MB +* Drive + * Add team drive support + * Reduce bandwidth by adding fields for partial responses - Martin Kristensen + * Implement --drive-shared-with-me flag to view shared with me files - Danny Tsai + * Add --drive-trashed-only to read only the files in the trash + * Remove obsolete --drive-full-list + * Add missing seek to start on retries of chunked uploads + * Fix stats accounting for upload + * Convert / in names to a unicode equivalent (／) + * Poll for Google Drive changes when mounted +* OneDrive + * Fix the uploading of files with spaces + * Fix initialization order for token renewer + * Display speeds accurately when uploading - Yoni Jah + * Swap to using http://localhost:53682/ as redirect URL - Michael Ledin + * Retry on token expired error, reset upload body on retry - Yoni Jah +* Google Cloud Storage + * Add ability to specify location and storage class via config and command line - thanks gdm85 + * Create container if necessary on server side copy + * Increase directory listing chunk to 1000 to increase performance + * Obtain a refresh token for GCS - Steven Lu +* Yandex + * Fix the name reported in log messages (was empty) + * Correct error return for listing empty directory +* Dropbox + * Rewritten to use the v2 API + * Now supports ModTime + * Can only set by uploading the file again + * If you uploaded with an old rclone, rclone may upload everything again + * Use `--size-only` or `--checksum` to avoid this + * Now supports the Dropbox content hashing scheme + * Now supports low level retries +* S3 + * Work around eventual consistency in bucket creation + * Create container if necessary on server side copy + * Add us-east-2 (Ohio) and eu-west-2 (London) S3 regions - Zahiar Ahmed +* Swift, Hubic + * Fix zero length directory markers showing in the subdirectory listing + * this caused lots of duplicate transfers + * Fix paged directory listings + * this caused duplicate directory errors + * Create container if necessary on server side copy + * Increase directory listing chunk to 1000 to increase performance + * Make sensible error if the user forgets the container +* SFTP + * Add support for using ssh key files + * Fix under Windows + * Fix ssh agent on Windows + * Adapt to latest version of library - Igor Kharin + +## v1.36 - 2017-03-18 + +* New Features + * SFTP remote (Jack Schmidt) + * Re-implement sync routine to work a directory at a time reducing memory usage + * Logging revamped to be more inline with rsync - now much quieter + * -v only shows transfers + * -vv is for full debug + * --syslog to log to syslog on capable platforms + * Implement --backup-dir and --suffix + * Implement --track-renames (initial implementation by Bjørn Erik Pedersen) + * Add time-based bandwidth limits (Lukas Loesche) + * rclone cryptcheck: checks integrity of crypt remotes + * Allow all config file variables and options to be set from environment variables + * Add --buffer-size parameter to control buffer size for copy + * Make --delete-after the default + * Add --ignore-checksum flag (fixed by Hisham Zarka) + * rclone check: Add --download flag to check all the data, not just hashes + * rclone cat: add --head, --tail, --offset, --count and --discard + * rclone config: when choosing from a list, allow the value to be entered too + * rclone config: allow rename and copy of remotes + * rclone obscure: for generating encrypted passwords for rclone's config (T.C. Ferguson) + * Comply with XDG Base Directory specification (Dario Giovannetti) + * this moves the default location of the config file in a backwards compatible way + * Release changes + * Ubuntu snap support (Dedsec1) + * Compile with go 1.8 + * MIPS/Linux big and little endian support +* Bug Fixes + * Fix copyto copying things to the wrong place if the destination dir didn't exist + * Fix parsing of remotes in moveto and copyto + * Fix --delete-before deleting files on copy + * Fix --files-from with an empty file copying everything + * Fix sync: don't update mod times if --dry-run set + * Fix MimeType propagation + * Fix filters to add ** rules to directory rules +* Local + * Implement -L, --copy-links flag to allow rclone to follow symlinks + * Open files in write only mode so rclone can write to an rclone mount + * Fix unnormalised unicode causing problems reading directories + * Fix interaction between -x flag and --max-depth +* Mount + * Implement proper directory handling (mkdir, rmdir, renaming) + * Make include and exclude filters apply to mount + * Implement read and write async buffers - control with --buffer-size + * Fix fsync on for directories + * Fix retry on network failure when reading off crypt +* Crypt + * Add --crypt-show-mapping to show encrypted file mapping + * Fix crypt writer getting stuck in a loop + * **IMPORTANT** this bug had the potential to cause data corruption when + * reading data from a network based remote and + * writing to a crypt on Google Drive + * Use the cryptcheck command to validate your data if you are concerned + * If syncing two crypt remotes, sync the unencrypted remote +* Amazon Drive + * Fix panics on Move (rename) + * Fix panic on token expiry +* B2 + * Fix inconsistent listings and rclone check + * Fix uploading empty files with go1.8 + * Constrain memory usage when doing multipart uploads + * Fix upload url not being refreshed properly +* Drive + * Fix Rmdir on directories with trashed files + * Fix "Ignoring unknown object" when downloading + * Add --drive-list-chunk + * Add --drive-skip-gdocs (Károly Oláh) +* OneDrive + * Implement Move + * Fix Copy + * Fix overwrite detection in Copy + * Fix waitForJob to parse errors correctly + * Use token renewer to stop auth errors on long uploads + * Fix uploading empty files with go1.8 +* Google Cloud Storage + * Fix depth 1 directory listings +* Yandex + * Fix single level directory listing +* Dropbox + * Normalise the case for single level directory listings + * Fix depth 1 listing +* S3 + * Added ca-central-1 region (Jon Yergatian) + +## v1.35 - 2017-01-02 + +* New Features + * moveto and copyto commands for choosing a destination name on copy/move + * rmdirs command to recursively delete empty directories + * Allow repeated --include/--exclude/--filter options + * Only show transfer stats on commands which transfer stuff + * show stats on any command using the `--stats` flag + * Allow overlapping directories in move when server side dir move is supported + * Add --stats-unit option - thanks Scott McGillivray +* Bug Fixes + * Fix the config file being overwritten when two rclones are running + * Make rclone lsd obey the filters properly + * Fix compilation on mips + * Fix not transferring files that don't differ in size + * Fix panic on nil retry/fatal error +* Mount + * Retry reads on error - should help with reliability a lot + * Report the modification times for directories from the remote + * Add bandwidth accounting and limiting (fixes --bwlimit) + * If --stats provided will show stats and which files are transferring + * Support R/W files if truncate is set. + * Implement statfs interface so df works + * Note that write is now supported on Amazon Drive + * Report number of blocks in a file - thanks Stefan Breunig +* Crypt + * Prevent the user pointing crypt at itself + * Fix failed to authenticate decrypted block errors + * these will now return the underlying unexpected EOF instead +* Amazon Drive + * Add support for server side move and directory move - thanks Stefan Breunig + * Fix nil pointer deref on size attribute +* B2 + * Use new prefix and delimiter parameters in directory listings + * This makes --max-depth 1 dir listings as used in mount much faster + * Reauth the account while doing uploads too - should help with token expiry +* Drive + * Make DirMove more efficient and complain about moving the root + * Create destination directory on Move() + +## v1.34 - 2016-11-06 + +* New Features + * Stop single file and `--files-from` operations iterating through the source bucket. + * Stop removing failed upload to cloud storage remotes + * Make ContentType be preserved for cloud to cloud copies + * Add support to toggle bandwidth limits via SIGUSR2 - thanks Marco Paganini + * `rclone check` shows count of hashes that couldn't be checked + * `rclone listremotes` command + * Support linux/arm64 build - thanks Fredrik Fornwall + * Remove `Authorization:` lines from `--dump-headers` output +* Bug Fixes + * Ignore files with control characters in the names + * Fix `rclone move` command + * Delete src files which already existed in dst + * Fix deletion of src file when dst file older + * Fix `rclone check` on crypted file systems + * Make failed uploads not count as "Transferred" + * Make sure high level retries show with `-q` + * Use a vendor directory with godep for repeatable builds +* `rclone mount` - FUSE + * Implement FUSE mount options + * `--no-modtime`, `--debug-fuse`, `--read-only`, `--allow-non-empty`, `--allow-root`, `--allow-other` + * `--default-permissions`, `--write-back-cache`, `--max-read-ahead`, `--umask`, `--uid`, `--gid` + * Add `--dir-cache-time` to control caching of directory entries + * Implement seek for files opened for read (useful for video players) + * with `-no-seek` flag to disable + * Fix crash on 32 bit ARM (alignment of 64 bit counter) + * ...and many more internal fixes and improvements! +* Crypt + * Don't show encrypted password in configurator to stop confusion +* Amazon Drive + * New wait for upload option `--acd-upload-wait-per-gb` + * upload timeouts scale by file size and can be disabled + * Add 502 Bad Gateway to list of errors we retry + * Fix overwriting a file with a zero length file + * Fix ACD file size warning limit - thanks Felix Bünemann +* Local + * Unix: implement `-x`/`--one-file-system` to stay on a single file system + * thanks Durval Menezes and Luiz Carlos Rumbelsperger Viana + * Windows: ignore the symlink bit on files + * Windows: Ignore directory based junction points +* B2 + * Make sure each upload has at least one upload slot - fixes strange upload stats + * Fix uploads when using crypt + * Fix download of large files (sha1 mismatch) + * Return error when we try to create a bucket which someone else owns + * Update B2 docs with Data usage, and Crypt section - thanks Tomasz Mazur +* S3 + * Command line and config file support for + * Setting/overriding ACL - thanks Radek Senfeld + * Setting storage class - thanks Asko Tamm +* Drive + * Make exponential backoff work exactly as per Google specification + * add `.epub`, `.odp` and `.tsv` as export formats. +* Swift + * Don't read metadata for directory marker objects + +## v1.33 - 2016-08-24 + +* New Features + * Implement encryption + * data encrypted in NACL secretbox format + * with optional file name encryption + * New commands + * rclone mount - implements FUSE mounting of remotes (EXPERIMENTAL) + * works on Linux, FreeBSD and OS X (need testers for the last 2!) + * rclone cat - outputs remote file or files to the terminal + * rclone genautocomplete - command to make a bash completion script for rclone + * Editing a remote using `rclone config` now goes through the wizard + * Compile with go 1.7 - this fixes rclone on macOS Sierra and on 386 processors + * Use cobra for sub commands and docs generation +* drive + * Document how to make your own client_id +* s3 + * User-configurable Amazon S3 ACL (thanks Radek Šenfeld) +* b2 + * Fix stats accounting for upload - no more jumping to 100% done + * On cleanup delete hide marker if it is the current file + * New B2 API endpoint (thanks Per Cederberg) + * Set maximum backoff to 5 Minutes +* onedrive + * Fix URL escaping in file names - eg uploading files with `+` in them. +* amazon cloud drive + * Fix token expiry during large uploads + * Work around 408 REQUEST_TIMEOUT and 504 GATEWAY_TIMEOUT errors +* local + * Fix filenames with invalid UTF-8 not being uploaded + * Fix problem with some UTF-8 characters on OS X + +## v1.32 - 2016-07-13 + +* Backblaze B2 + * Fix upload of files large files not in root + +## v1.31 - 2016-07-13 + +* New Features + * Reduce memory on sync by about 50% + * Implement --no-traverse flag to stop copy traversing the destination remote. + * This can be used to reduce memory usage down to the smallest possible. + * Useful to copy a small number of files into a large destination folder. + * Implement cleanup command for emptying trash / removing old versions of files + * Currently B2 only + * Single file handling improved + * Now copied with --files-from + * Automatically sets --no-traverse when copying a single file + * Info on using installing with ansible - thanks Stefan Weichinger + * Implement --no-update-modtime flag to stop rclone fixing the remote modified times. +* Bug Fixes + * Fix move command - stop it running for overlapping Fses - this was causing data loss. +* Local + * Fix incomplete hashes - this was causing problems for B2. +* Amazon Drive + * Rename Amazon Cloud Drive to Amazon Drive - no changes to config file needed. +* Swift + * Add support for non-default project domain - thanks Antonio Messina. +* S3 + * Add instructions on how to use rclone with minio. + * Add ap-northeast-2 (Seoul) and ap-south-1 (Mumbai) regions. + * Skip setting the modified time for objects > 5GB as it isn't possible. +* Backblaze B2 + * Add --b2-versions flag so old versions can be listed and retreived. + * Treat 403 errors (eg cap exceeded) as fatal. + * Implement cleanup command for deleting old file versions. + * Make error handling compliant with B2 integrations notes. + * Fix handling of token expiry. + * Implement --b2-test-mode to set `X-Bz-Test-Mode` header. + * Set cutoff for chunked upload to 200MB as per B2 guidelines. + * Make upload multi-threaded. +* Dropbox + * Don't retry 461 errors. + +## v1.30 - 2016-06-18 + +* New Features + * Directory listing code reworked for more features and better error reporting (thanks to Klaus Post for help). This enables + * Directory include filtering for efficiency + * --max-depth parameter + * Better error reporting + * More to come + * Retry more errors + * Add --ignore-size flag - for uploading images to onedrive + * Log -v output to stdout by default + * Display the transfer stats in more human readable form + * Make 0 size files specifiable with `--max-size 0b` + * Add `b` suffix so we can specify bytes in --bwlimit, --min-size etc + * Use "password:" instead of "password>" prompt - thanks Klaus Post and Leigh Klotz +* Bug Fixes + * Fix retry doing one too many retries +* Local + * Fix problems with OS X and UTF-8 characters +* Amazon Drive + * Check a file exists before uploading to help with 408 Conflict errors + * Reauth on 401 errors - this has been causing a lot of problems + * Work around spurious 403 errors + * Restart directory listings on error +* Google Drive + * Check a file exists before uploading to help with duplicates + * Fix retry of multipart uploads +* Backblaze B2 + * Implement large file uploading +* S3 + * Add AES256 server-side encryption for - thanks Justin R. Wilson +* Google Cloud Storage + * Make sure we don't use conflicting content types on upload + * Add service account support - thanks Michal Witkowski +* Swift + * Add auth version parameter + * Add domain option for openstack (v3 auth) - thanks Fabian Ruff + +## v1.29 - 2016-04-18 + +* New Features + * Implement `-I, --ignore-times` for unconditional upload + * Improve `dedupe`command + * Now removes identical copies without asking + * Now obeys `--dry-run` + * Implement `--dedupe-mode` for non interactive running + * `--dedupe-mode interactive` - interactive the default. + * `--dedupe-mode skip` - removes identical files then skips anything left. + * `--dedupe-mode first` - removes identical files then keeps the first one. + * `--dedupe-mode newest` - removes identical files then keeps the newest one. + * `--dedupe-mode oldest` - removes identical files then keeps the oldest one. + * `--dedupe-mode rename` - removes identical files then renames the rest to be different. +* Bug fixes + * Make rclone check obey the `--size-only` flag. + * Use "application/octet-stream" if discovered mime type is invalid. + * Fix missing "quit" option when there are no remotes. +* Google Drive + * Increase default chunk size to 8 MB - increases upload speed of big files + * Speed up directory listings and make more reliable + * Add missing retries for Move and DirMove - increases reliability + * Preserve mime type on file update +* Backblaze B2 + * Enable mod time syncing + * This means that B2 will now check modification times + * It will upload new files to update the modification times + * (there isn't an API to just set the mod time.) + * If you want the old behaviour use `--size-only`. + * Update API to new version + * Fix parsing of mod time when not in metadata +* Swift/Hubic + * Don't return an MD5SUM for static large objects +* S3 + * Fix uploading files bigger than 50GB + +## v1.28 - 2016-03-01 + +* New Features + * Configuration file encryption - thanks Klaus Post + * Improve `rclone config` adding more help and making it easier to understand + * Implement `-u`/`--update` so creation times can be used on all remotes + * Implement `--low-level-retries` flag + * Optionally disable gzip compression on downloads with `--no-gzip-encoding` +* Bug fixes + * Don't make directories if `--dry-run` set + * Fix and document the `move` command + * Fix redirecting stderr on unix-like OSes when using `--log-file` + * Fix `delete` command to wait until all finished - fixes missing deletes. +* Backblaze B2 + * Use one upload URL per go routine fixes `more than one upload using auth token` + * Add pacing, retries and reauthentication - fixes token expiry problems + * Upload without using a temporary file from local (and remotes which support SHA1) + * Fix reading metadata for all files when it shouldn't have been +* Drive + * Fix listing drive documents at root + * Disable copy and move for Google docs +* Swift + * Fix uploading of chunked files with non ASCII characters + * Allow setting of `storage_url` in the config - thanks Xavier Lucas +* S3 + * Allow IAM role and credentials from environment variables - thanks Brian Stengaard + * Allow low privilege users to use S3 (check if directory exists during Mkdir) - thanks Jakub Gedeon +* Amazon Drive + * Retry on more things to make directory listings more reliable + +## v1.27 - 2016-01-31 + +* New Features + * Easier headless configuration with `rclone authorize` + * Add support for multiple hash types - we now check SHA1 as well as MD5 hashes. + * `delete` command which does obey the filters (unlike `purge`) + * `dedupe` command to deduplicate a remote. Useful with Google Drive. + * Add `--ignore-existing` flag to skip all files that exist on destination. + * Add `--delete-before`, `--delete-during`, `--delete-after` flags. + * Add `--memprofile` flag to debug memory use. + * Warn the user about files with same name but different case + * Make `--include` rules add their implict exclude * at the end of the filter list + * Deprecate compiling with go1.3 +* Amazon Drive + * Fix download of files > 10 GB + * Fix directory traversal ("Next token is expired") for large directory listings + * Remove 409 conflict from error codes we will retry - stops very long pauses +* Backblaze B2 + * SHA1 hashes now checked by rclone core +* Drive + * Add `--drive-auth-owner-only` to only consider files owned by the user - thanks Björn Harrtell + * Export Google documents +* Dropbox + * Make file exclusion error controllable with -q +* Swift + * Fix upload from unprivileged user. +* S3 + * Fix updating of mod times of files with `+` in. +* Local + * Add local file system option to disable UNC on Windows. + +## v1.26 - 2016-01-02 + +* New Features + * Yandex storage backend - thank you Dmitry Burdeev ("dibu") + * Implement Backblaze B2 storage backend + * Add --min-age and --max-age flags - thank you Adriano Aurélio Meirelles + * Make ls/lsl/md5sum/size/check obey includes and excludes +* Fixes + * Fix crash in http logging + * Upload releases to github too +* Swift + * Fix sync for chunked files +* OneDrive + * Re-enable server side copy + * Don't mask HTTP error codes with JSON decode error +* S3 + * Fix corrupting Content-Type on mod time update (thanks Joseph Spurrier) + +## v1.25 - 2015-11-14 + +* New features + * Implement Hubic storage system +* Fixes + * Fix deletion of some excluded files without --delete-excluded + * This could have deleted files unexpectedly on sync + * Always check first with `--dry-run`! +* Swift + * Stop SetModTime losing metadata (eg X-Object-Manifest) + * This could have caused data loss for files > 5GB in size + * Use ContentType from Object to avoid lookups in listings +* OneDrive + * disable server side copy as it seems to be broken at Microsoft + +## v1.24 - 2015-11-07 + +* New features + * Add support for Microsoft OneDrive + * Add `--no-check-certificate` option to disable server certificate verification + * Add async readahead buffer for faster transfer of big files +* Fixes + * Allow spaces in remotes and check remote names for validity at creation time + * Allow '&' and disallow ':' in Windows filenames. +* Swift + * Ignore directory marker objects where appropriate - allows working with Hubic + * Don't delete the container if fs wasn't at root +* S3 + * Don't delete the bucket if fs wasn't at root +* Google Cloud Storage + * Don't delete the bucket if fs wasn't at root + +## v1.23 - 2015-10-03 + +* New features + * Implement `rclone size` for measuring remotes +* Fixes + * Fix headless config for drive and gcs + * Tell the user they should try again if the webserver method failed + * Improve output of `--dump-headers` +* S3 + * Allow anonymous access to public buckets +* Swift + * Stop chunked operations logging "Failed to read info: Object Not Found" + * Use Content-Length on uploads for extra reliability + +## v1.22 - 2015-09-28 + +* Implement rsync like include and exclude flags +* swift + * Support files > 5GB - thanks Sergey Tolmachev + +## v1.21 - 2015-09-22 + +* New features + * Display individual transfer progress + * Make lsl output times in localtime +* Fixes + * Fix allowing user to override credentials again in Drive, GCS and ACD +* Amazon Drive + * Implement compliant pacing scheme +* Google Drive + * Make directory reads concurrent for increased speed. + +## v1.20 - 2015-09-15 + +* New features + * Amazon Drive support + * Oauth support redone - fix many bugs and improve usability + * Use "golang.org/x/oauth2" as oauth libary of choice + * Improve oauth usability for smoother initial signup + * drive, googlecloudstorage: optionally use auto config for the oauth token + * Implement --dump-headers and --dump-bodies debug flags + * Show multiple matched commands if abbreviation too short + * Implement server side move where possible +* local + * Always use UNC paths internally on Windows - fixes a lot of bugs +* dropbox + * force use of our custom transport which makes timeouts work +* Thanks to Klaus Post for lots of help with this release + +## v1.19 - 2015-08-28 + +* New features + * Server side copies for s3/swift/drive/dropbox/gcs + * Move command - uses server side copies if it can + * Implement --retries flag - tries 3 times by default + * Build for plan9/amd64 and solaris/amd64 too +* Fixes + * Make a current version download with a fixed URL for scripting + * Ignore rmdir in limited fs rather than throwing error +* dropbox + * Increase chunk size to improve upload speeds massively + * Issue an error message when trying to upload bad file name + +## v1.18 - 2015-08-17 + +* drive + * Add `--drive-use-trash` flag so rclone trashes instead of deletes + * Add "Forbidden to download" message for files with no downloadURL +* dropbox + * Remove datastore + * This was deprecated and it caused a lot of problems + * Modification times and MD5SUMs no longer stored + * Fix uploading files > 2GB +* s3 + * use official AWS SDK from github.com/aws/aws-sdk-go + * **NB** will most likely require you to delete and recreate remote + * enable multipart upload which enables files > 5GB + * tested with Ceph / RadosGW / S3 emulation + * many thanks to Sam Liston and Brian Haymore at the [Utah Center for High Performance Computing](https://www.chpc.utah.edu/) for a Ceph test account +* misc + * Show errors when reading the config file + * Do not print stats in quiet mode - thanks Leonid Shalupov + * Add FAQ + * Fix created directories not obeying umask + * Linux installation instructions - thanks Shimon Doodkin + +## v1.17 - 2015-06-14 + +* dropbox: fix case insensitivity issues - thanks Leonid Shalupov + +## v1.16 - 2015-06-09 + +* Fix uploading big files which was causing timeouts or panics +* Don't check md5sum after download with --size-only + +## v1.15 - 2015-06-06 + +* Add --checksum flag to only discard transfers by MD5SUM - thanks Alex Couper +* Implement --size-only flag to sync on size not checksum & modtime +* Expand docs and remove duplicated information +* Document rclone's limitations with directories +* dropbox: update docs about case insensitivity + +## v1.14 - 2015-05-21 + +* local: fix encoding of non utf-8 file names - fixes a duplicate file problem +* drive: docs about rate limiting +* google cloud storage: Fix compile after API change in "google.golang.org/api/storage/v1" + +## v1.13 - 2015-05-10 + +* Revise documentation (especially sync) +* Implement --timeout and --conntimeout +* s3: ignore etags from multipart uploads which aren't md5sums + +## v1.12 - 2015-03-15 + +* drive: Use chunked upload for files above a certain size +* drive: add --drive-chunk-size and --drive-upload-cutoff parameters +* drive: switch to insert from update when a failed copy deletes the upload +* core: Log duplicate files if they are detected + +## v1.11 - 2015-03-04 + +* swift: add region parameter +* drive: fix crash on failed to update remote mtime +* In remote paths, change native directory separators to / +* Add synchronization to ls/lsl/lsd output to stop corruptions +* Ensure all stats/log messages to go stderr +* Add --log-file flag to log everything (including panics) to file +* Make it possible to disable stats printing with --stats=0 +* Implement --bwlimit to limit data transfer bandwidth + +## v1.10 - 2015-02-12 + +* s3: list an unlimited number of items +* Fix getting stuck in the configurator + +## v1.09 - 2015-02-07 + +* windows: Stop drive letters (eg C:) getting mixed up with remotes (eg drive:) +* local: Fix directory separators on Windows +* drive: fix rate limit exceeded errors + +## v1.08 - 2015-02-04 + +* drive: fix subdirectory listing to not list entire drive +* drive: Fix SetModTime +* dropbox: adapt code to recent library changes + +## v1.07 - 2014-12-23 + +* google cloud storage: fix memory leak + +## v1.06 - 2014-12-12 + +* Fix "Couldn't find home directory" on OSX +* swift: Add tenant parameter +* Use new location of Google API packages + +## v1.05 - 2014-08-09 + +* Improved tests and consequently lots of minor fixes +* core: Fix race detected by go race detector +* core: Fixes after running errcheck +* drive: reset root directory on Rmdir and Purge +* fs: Document that Purger returns error on empty directory, test and fix +* google cloud storage: fix ListDir on subdirectory +* google cloud storage: re-read metadata in SetModTime +* s3: make reading metadata more reliable to work around eventual consistency problems +* s3: strip trailing / from ListDir() +* swift: return directories without / in ListDir + +## v1.04 - 2014-07-21 + +* google cloud storage: Fix crash on Update + +## v1.03 - 2014-07-20 + +* swift, s3, dropbox: fix updated files being marked as corrupted +* Make compile with go 1.1 again + +## v1.02 - 2014-07-19 + +* Implement Dropbox remote +* Implement Google Cloud Storage remote +* Verify Md5sums and Sizes after copies +* Remove times from "ls" command - lists sizes only +* Add add "lsl" - lists times and sizes +* Add "md5sum" command + +## v1.01 - 2014-07-04 + +* drive: fix transfer of big files using up lots of memory + +## v1.00 - 2014-07-03 + +* drive: fix whole second dates + +## v0.99 - 2014-06-26 + +* Fix --dry-run not working +* Make compatible with go 1.1 + +## v0.98 - 2014-05-30 + +* s3: Treat missing Content-Length as 0 for some ceph installations +* rclonetest: add file with a space in + +## v0.97 - 2014-05-05 + +* Implement copying of single files +* s3 & swift: support paths inside containers/buckets + +## v0.96 - 2014-04-24 + +* drive: Fix multiple files of same name being created +* drive: Use o.Update and fs.Put to optimise transfers +* Add version number, -V and --version + +## v0.95 - 2014-03-28 + +* rclone.org: website, docs and graphics +* drive: fix path parsing + +## v0.94 - 2014-03-27 + +* Change remote format one last time +* GNU style flags + +## v0.93 - 2014-03-16 + +* drive: store token in config file +* cross compile other versions +* set strict permissions on config file + +## v0.92 - 2014-03-15 + +* Config fixes and --config option + +## v0.91 - 2014-03-15 + +* Make config file + +## v0.90 - 2013-06-27 + +* Project named rclone + +## v0.00 - 2012-11-18 + +* Project started + +Bugs and Limitations +-------------------- + +### Empty directories are left behind / not created ## + +With remotes that have a concept of directory, eg Local and Drive, +empty directories may be left behind, or not created when one was +expected. + +This is because rclone doesn't have a concept of a directory - it only +works on objects. Most of the object storage systems can't actually +store a directory so there is nowhere for rclone to store anything +about directories. + +You can work round this to some extent with the`purge` command which +will delete everything under the path, **inluding** empty directories. + +This may be fixed at some point in +[Issue #100](https://github.com/ncw/rclone/issues/100) + +### Directory timestamps aren't preserved ## + +For the same reason as the above, rclone doesn't have a concept of a +directory - it only works on objects, therefore it can't preserve the +timestamps of directories. + +Frequently Asked Questions +-------------------------- + +### Do all cloud storage systems support all rclone commands ### + +Yes they do. All the rclone commands (eg `sync`, `copy` etc) will +work on all the remote storage systems. + +### Can I copy the config from one machine to another ### + +Sure! Rclone stores all of its config in a single file. If you want +to find this file, the simplest way is to run `rclone -h` and look at +the help for the `--config` flag which will tell you where it is. + +See the [remote setup docs](https://rclone.org/remote_setup/) for more info. + +### How do I configure rclone on a remote / headless box with no browser? ### + +This has now been documented in its own [remote setup page](https://rclone.org/remote_setup/). + +### Can rclone sync directly from drive to s3 ### + +Rclone can sync between two remote cloud storage systems just fine. + +Note that it effectively downloads the file and uploads it again, so +the node running rclone would need to have lots of bandwidth. + +The syncs would be incremental (on a file by file basis). + +Eg + + rclone sync drive:Folder s3:bucket + + +### Using rclone from multiple locations at the same time ### + +You can use rclone from multiple places at the same time if you choose +different subdirectory for the output, eg + +``` +Server A> rclone sync /tmp/whatever remote:ServerA +Server B> rclone sync /tmp/whatever remote:ServerB +``` + +If you sync to the same directory then you should use rclone copy +otherwise the two rclones may delete each others files, eg + +``` +Server A> rclone copy /tmp/whatever remote:Backup +Server B> rclone copy /tmp/whatever remote:Backup +``` + +The file names you upload from Server A and Server B should be +different in this case, otherwise some file systems (eg Drive) may +make duplicates. + +### Why doesn't rclone support partial transfers / binary diffs like rsync? ### + +Rclone stores each file you transfer as a native object on the remote +cloud storage system. This means that you can see the files you +upload as expected using alternative access methods (eg using the +Google Drive web interface). There is a 1:1 mapping between files on +your hard disk and objects created in the cloud storage system. + +Cloud storage systems (at least none I've come across yet) don't +support partially uploading an object. You can't take an existing +object, and change some bytes in the middle of it. + +It would be possible to make a sync system which stored binary diffs +instead of whole objects like rclone does, but that would break the +1:1 mapping of files on your hard disk to objects in the remote cloud +storage system. + +All the cloud storage systems support partial downloads of content, so +it would be possible to make partial downloads work. However to make +this work efficiently this would require storing a significant amount +of metadata, which breaks the desired 1:1 mapping of files to objects. + +### Can rclone do bi-directional sync? ### + +No, not at present. rclone only does uni-directional sync from A -> +B. It may do in the future though since it has all the primitives - it +just requires writing the algorithm to do it. + +### Can I use rclone with an HTTP proxy? ### + +Yes. rclone will use the environment variables `HTTP_PROXY`, +`HTTPS_PROXY` and `NO_PROXY`, similar to cURL and other programs. + +`HTTPS_PROXY` takes precedence over `HTTP_PROXY` for https requests. + +The environment values may be either a complete URL or a "host[:port]", +in which case the "http" scheme is assumed. + +The `NO_PROXY` allows you to disable the proxy for specific hosts. +Hosts must be comma separated, and can contain domains or parts. +For instance "foo.com" also matches "bar.foo.com". + +### Rclone gives x509: failed to load system roots and no roots provided error ### + +This means that `rclone` can't file the SSL root certificates. Likely +you are running `rclone` on a NAS with a cut-down Linux OS, or +possibly on Solaris. + +Rclone (via the Go runtime) tries to load the root certificates from +these places on Linux. + + "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc. + "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL + "/etc/ssl/ca-bundle.pem", // OpenSUSE + "/etc/pki/tls/cacert.pem", // OpenELEC + +So doing something like this should fix the problem. It also sets the +time which is important for SSL to work properly. + +``` +mkdir -p /etc/ssl/certs/ +curl -o /etc/ssl/certs/ca-certificates.crt https://raw.githubusercontent.com/bagder/ca-bundle/master/ca-bundle.crt +ntpclient -s -h pool.ntp.org +``` + +The two environment variables `SSL_CERT_FILE` and `SSL_CERT_DIR`, mentioned in the [x509 pacakge](https://godoc.org/crypto/x509), +provide an additional way to provide the SSL root certificates. + +Note that you may need to add the `--insecure` option to the `curl` command line if it doesn't work without. + +``` +curl --insecure -o /etc/ssl/certs/ca-certificates.crt https://raw.githubusercontent.com/bagder/ca-bundle/master/ca-bundle.crt +``` + +### Rclone gives Failed to load config file: function not implemented error ### + +Likely this means that you are running rclone on Linux version not +supported by the go runtime, ie earlier than version 2.6.23. + +See the [system requirements section in the go install +docs](https://golang.org/doc/install) for full details. + +### All my uploaded docx/xlsx/pptx files appear as archive/zip ### + +This is caused by uploading these files from a Windows computer which +hasn't got the Microsoft Office suite installed. The easiest way to +fix is to install the Word viewer and the Microsoft Office +Compatibility Pack for Word, Excel, and PowerPoint 2007 and later +versions' file formats + +### tcp lookup some.domain.com no such host ### + +This happens when rclone cannot resolve a domain. Please check that +your DNS setup is generally working, e.g. + +``` +# both should print a long list of possible IP addresses +dig www.googleapis.com # resolve using your default DNS +dig www.googleapis.com @8.8.8.8 # resolve with Google's DNS server +``` + +If you are using `systemd-resolved` (default on Arch Linux), ensure it +is at version 233 or higher. Previous releases contain a bug which +causes not all domains to be resolved properly. + +Additionally with the `GODEBUG=netdns=` environment variable the Go +resolver decision can be influenced. This also allows to resolve certain +issues with DNS resolution. See the [name resolution section in the go docs](https://golang.org/pkg/net/#hdr-Name_Resolution). + +License +------- + +This is free software under the terms of MIT the license (check the +COPYING file included with the source code). + +``` +Copyright (C) 2012 by Nick Craig-Wood https://www.craig-wood.com/nick/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +``` + +Authors +------- + + * Nick Craig-Wood + +Contributors +------------ + + * Alex Couper + * Leonid Shalupov + * Shimon Doodkin + * Colin Nicholson + * Klaus Post + * Sergey Tolmachev + * Adriano Aurélio Meirelles + * C. Bess + * Dmitry Burdeev + * Joseph Spurrier + * Björn Harrtell + * Xavier Lucas + * Werner Beroux + * Brian Stengaard + * Jakub Gedeon + * Jim Tittsler + * Michal Witkowski + * Fabian Ruff + * Leigh Klotz + * Romain Lapray + * Justin R. Wilson + * Antonio Messina + * Stefan G. Weichinger + * Per Cederberg + * Radek Šenfeld + * Fredrik Fornwall + * Asko Tamm + * xor-zz + * Tomasz Mazur + * Marco Paganini + * Felix Bünemann + * Durval Menezes + * Luiz Carlos Rumbelsperger Viana + * Stefan Breunig + * Alishan Ladhani + * 0xJAKE <0xJAKE@users.noreply.github.com> + * Thibault Molleman + * Scott McGillivray + * Bjørn Erik Pedersen + * Lukas Loesche + * emyarod + * T.C. Ferguson + * Brandur + * Dario Giovannetti + * Károly Oláh + * Jon Yergatian + * Jack Schmidt + * Dedsec1 + * Hisham Zarka + * Jérôme Vizcaino + * Mike Tesch + * Marvin Watson + * Danny Tsai + * Yoni Jah + * Stephen Harris + * Ihor Dvoretskyi + * Jon Craton + * Hraban Luyat + * Michael Ledin + * Martin Kristensen + * Too Much IO + * Anisse Astier + * Zahiar Ahmed + * Igor Kharin + * Bill Zissimopoulos + * Bob Potter + * Steven Lu + * Sjur Fredriksen + * Ruwbin + * Fabian Möller + * Edward Q. Bridges + * Vasiliy Tolstov + * Harshavardhana + * sainaen + * gdm85 + * Yaroslav Halchenko + * John Papandriopoulos + * Zhiming Wang + * Andy Pilate + * Oliver Heyme + * wuyu + * Andrei Dragomir + * Christian Brüggemann + * Alex McGrath Kraak + * bpicode + * Daniel Jagszent + * Josiah White + * Ishuah Kariuki + * Jan Varho + * Girish Ramakrishnan + * LingMan + * Jacob McNamee + * jersou + * thierry + * Simon Leinen + * Dan Dascalescu + * Jason Rose + * Andrew Starr-Bochicchio + * John Leach + * Corban Raun + * Pierre Carlson + * Ernest Borowski + * Remus Bunduc + * Iakov Davydov + * Jakub Tasiemski + * David Minor + * Tim Cooijmans + * Laurence + * Giovanni Pizzi + * Filip Bartodziej + * Jon Fautley + * lewapm <32110057+lewapm@users.noreply.github.com> + * Yassine Imounachen + * Chris Redekop + * Jon Fautley + * Will Gunn + * Lucas Bremgartner + * Jody Frankowski + * Andreas Roussos + * nbuchanan + * Durval Menezes + * Victor + * Mateusz + * Daniel Loader + * David0rk + * Alexander Neumann + * Giri Badanahatti + * Leo R. Lundgren + * wolfv + * Dave Pedu + * Stefan Lindblom + * seuffert + * gbadanahatti <37121690+gbadanahatti@users.noreply.github.com> + * Keith Goldfarb + * Steve Kriss + * Chih-Hsuan Yen + * Alexander Neumann + * Matt Holt + * Eri Bastos + * Michael P. Dubner + * Antoine GIRARD + * Mateusz Piotrowski + * Animosity022 + * Peter Baumgartner + * Craig Rachel + * Michael G. Noll + * hensur + * Oliver Heyme + * Richard Yang + * Piotr Oleszczyk + * Rodrigo + * NoLooseEnds + * Jakub Karlicek + * John Clayton + * Kasper Byrdal Nielsen + * Benjamin Joseph Dag + * themylogin + * Onno Zweers + * Jasper Lievisse Adriaanse + * sandeepkru + * HerrH + * Andrew <4030760+sparkyman215@users.noreply.github.com> + * dan smith + * Oleg Kovalov + * Ruben Vandamme + * Cnly + * Andres Alvarez <1671935+kir4h@users.noreply.github.com> + * reddi1 + * Matt Tucker + * Sebastian Bünger + * Martin Polden + * Alex Chen + * Denis + * bsteiss <35940619+bsteiss@users.noreply.github.com> + +# Contact the rclone project # + +## Forum ## + +Forum for general discussions and questions: + + * https://forum.rclone.org + +## Gitub project ## + +The project website is at: + + * https://github.com/ncw/rclone + +There you can file bug reports, ask for help or contribute pull +requests. + +## Google+ ## + +Rclone has a Google+ page which announcements are posted to + + * Google+ page for general comments + +## Twitter ## + +You can also follow me on twitter for rclone announcements + + * [@njcw](https://twitter.com/njcw) + +## Email ## + +Or if all else fails or you want to ask something private or +confidential email [Nick Craig-Wood](mailto:nick@craig-wood.com) + diff --git a/.rclone_repo/MANUAL.txt b/.rclone_repo/MANUAL.txt new file mode 100755 index 0000000..1b2d699 --- /dev/null +++ b/.rclone_repo/MANUAL.txt @@ -0,0 +1,13287 @@ +rclone(1) User Manual +Nick Craig-Wood +Sep 07, 2018 + + + +RCLONE + + +[Logo] + +Rclone is a command line program to sync files and directories to and +from: + +- Amazon Drive (See note) +- Amazon S3 +- Backblaze B2 +- Box +- Ceph +- DigitalOcean Spaces +- Dreamhost +- Dropbox +- FTP +- Google Cloud Storage +- Google Drive +- HTTP +- Hubic +- Jottacloud +- IBM COS S3 +- Memset Memstore +- Mega +- Microsoft Azure Blob Storage +- Microsoft OneDrive +- Minio +- Nextcloud +- OVH +- OpenDrive +- Openstack Swift +- Oracle Cloud Storage +- ownCloud +- pCloud +- put.io +- QingStor +- Rackspace Cloud Files +- SFTP +- Wasabi +- WebDAV +- Yandex Disk +- The local filesystem + +Features + +- MD5/SHA1 hashes checked at all times for file integrity +- Timestamps preserved on files +- Partial syncs supported on a whole file basis +- Copy mode to just copy new/changed files +- Sync (one way) mode to make a directory identical +- Check mode to check for file hash equality +- Can sync to and from network, eg two different cloud accounts +- Optional encryption (Crypt) +- Optional cache (Cache) +- Optional FUSE mount (rclone mount) + +Links + +- Home page +- Github project page for source and bug tracker +- Rclone Forum +- Google+ page +- Downloads + + + +INSTALL + + +Rclone is a Go program and comes as a single binary file. + + +Quickstart + +- Download the relevant binary. +- Extract the rclone or rclone.exe binary from the archive +- Run rclone config to setup. See rclone config docs for more details. + +See below for some expanded Linux / macOS instructions. + +See the Usage section of the docs for how to use rclone, or run +rclone -h. + + +Script installation + +To install rclone on Linux/macOS/BSD systems, run: + + curl https://rclone.org/install.sh | sudo bash + +For beta installation, run: + + curl https://rclone.org/install.sh | sudo bash -s beta + +Note that this script checks the version of rclone installed first and +won't re-download if not needed. + + +Linux installation from precompiled binary + +Fetch and unpack + + curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip + unzip rclone-current-linux-amd64.zip + cd rclone-*-linux-amd64 + +Copy binary file + + sudo cp rclone /usr/bin/ + sudo chown root:root /usr/bin/rclone + sudo chmod 755 /usr/bin/rclone + +Install manpage + + sudo mkdir -p /usr/local/share/man/man1 + sudo cp rclone.1 /usr/local/share/man/man1/ + sudo mandb + +Run rclone config to setup. See rclone config docs for more details. + + rclone config + + +macOS installation from precompiled binary + +Download the latest version of rclone. + + cd && curl -O https://downloads.rclone.org/rclone-current-osx-amd64.zip + +Unzip the download and cd to the extracted folder. + + unzip -a rclone-current-osx-amd64.zip && cd rclone-*-osx-amd64 + +Move rclone to your $PATH. You will be prompted for your password. + + sudo mkdir -p /usr/local/bin + sudo mv rclone /usr/local/bin/ + +(the mkdir command is safe to run, even if the directory already +exists). + +Remove the leftover files. + + cd .. && rm -rf rclone-*-osx-amd64 rclone-current-osx-amd64.zip + +Run rclone config to setup. See rclone config docs for more details. + + rclone config + + +Install from source + +Make sure you have at least Go 1.7 installed. Download go if necessary. +The latest release is recommended. Then + + git clone https://github.com/ncw/rclone.git + cd rclone + go build + ./rclone version + +You can also build and install rclone in the GOPATH (which defaults to +~/go) with: + + go get -u -v github.com/ncw/rclone + +and this will build the binary in $GOPATH/bin (~/go/bin/rclone by +default) after downloading the source to +$GOPATH/src/github.com/ncw/rclone (~/go/src/github.com/ncw/rclone by +default). + + +Installation with Ansible + +This can be done with Stefan Weichinger's ansible role. + +Instructions + +1. git clone https://github.com/stefangweichinger/ansible-rclone.git + into your local roles-directory +2. add the role to the hosts you want rclone installed to: + +  + - hosts: rclone-hosts + roles: + - rclone + + +Configure + +First, you'll need to configure rclone. As the object storage systems +have quite complicated authentication these are kept in a config file. +(See the --config entry for how to find the config file and choose its +location.) + +The easiest way to make the config is to run rclone with the config +option: + + rclone config + +See the following for detailed instructions for + +- Alias +- Amazon Drive +- Amazon S3 +- Backblaze B2 +- Box +- Cache +- Crypt - to encrypt other remotes +- DigitalOcean Spaces +- Dropbox +- FTP +- Google Cloud Storage +- Google Drive +- HTTP +- Hubic +- Jottacloud +- Mega +- Microsoft Azure Blob Storage +- Microsoft OneDrive +- Openstack Swift / Rackspace Cloudfiles / Memset Memstore +- OpenDrive +- Pcloud +- QingStor +- SFTP +- WebDAV +- Yandex Disk +- The local filesystem + + +Usage + +Rclone syncs a directory tree from one storage system to another. + +Its syntax is like this + + Syntax: [options] subcommand + +Source and destination paths are specified by the name you gave the +storage system in the config file then the sub path, eg "drive:myfolder" +to look at "myfolder" in Google drive. + +You can define as many storage paths as you like in the config file. + + +Subcommands + +rclone uses a system of subcommands. For example + + rclone ls remote:path # lists a re + rclone copy /local/path remote:path # copies /local/path to the remote + rclone sync /local/path remote:path # syncs /local/path to the remote + + +rclone config + +Enter an interactive configuration session. + +Synopsis + +Enter an interactive configuration session where you can setup new +remotes and manage existing ones. You may also set or remove a password +to protect your configuration. + + rclone config [flags] + +Options + + -h, --help help for config + + +rclone copy + +Copy files from source to dest, skipping already copied + +Synopsis + +Copy the source to the destination. Doesn't transfer unchanged files, +testing by size and modification time or MD5SUM. Doesn't delete files +from the destination. + +Note that it is always the contents of the directory that is synced, not +the directory so when source:path is a directory, it's the contents of +source:path that are copied, not the directory name and contents. + +If dest:path doesn't exist, it is created and the source:path contents +go there. + +For example + + rclone copy source:sourcepath dest:destpath + +Let's say there are two files in sourcepath + + sourcepath/one.txt + sourcepath/two.txt + +This copies them to + + destpath/one.txt + destpath/two.txt + +Not to + + destpath/sourcepath/one.txt + destpath/sourcepath/two.txt + +If you are familiar with rsync, rclone always works as if you had +written a trailing / - meaning "copy the contents of this directory". +This applies to all commands and whether you are talking about the +source or destination. + + rclone copy source:path dest:path [flags] + +Options + + -h, --help help for copy + + +rclone sync + +Make source and dest identical, modifying destination only. + +Synopsis + +Sync the source to the destination, changing the destination only. +Doesn't transfer unchanged files, testing by size and modification time +or MD5SUM. Destination is updated to match source, including deleting +files if necessary. + +IMPORTANT: Since this can cause data loss, test first with the --dry-run +flag to see exactly what would be copied and deleted. + +Note that files in the destination won't be deleted if there were any +errors at any point. + +It is always the contents of the directory that is synced, not the +directory so when source:path is a directory, it's the contents of +source:path that are copied, not the directory name and contents. See +extended explanation in the copy command above if unsure. + +If dest:path doesn't exist, it is created and the source:path contents +go there. + + rclone sync source:path dest:path [flags] + +Options + + -h, --help help for sync + + +rclone move + +Move files from source to dest. + +Synopsis + +Moves the contents of the source directory to the destination directory. +Rclone will error if the source and destination overlap and the remote +does not support a server side directory move operation. + +If no filters are in use and if possible this will server side move +source:path into dest:path. After this source:path will no longer longer +exist. + +Otherwise for each file in source:path selected by the filters (if any) +this will move it into dest:path. If possible a server side move will be +used, otherwise it will copy it (server side if possible) into dest:path +then delete the original (if no errors on copy) in source:path. + +If you want to delete empty source directories after move, use the +--delete-empty-src-dirs flag. + +IMPORTANT: Since this can cause data loss, test first with the --dry-run +flag. + + rclone move source:path dest:path [flags] + +Options + + --delete-empty-src-dirs Delete empty source dirs after move + -h, --help help for move + + +rclone delete + +Remove the contents of path. + +Synopsis + +Remove the contents of path. Unlike purge it obeys include/exclude +filters so can be used to selectively delete files. + +Eg delete all files bigger than 100MBytes + +Check what would be deleted first (use either) + + rclone --min-size 100M lsl remote:path + rclone --dry-run --min-size 100M delete remote:path + +Then delete + + rclone --min-size 100M delete remote:path + +That reads "delete everything with a minimum size of 100 MB", hence +delete all files bigger than 100MBytes. + + rclone delete remote:path [flags] + +Options + + -h, --help help for delete + + +rclone purge + +Remove the path and all of its contents. + +Synopsis + +Remove the path and all of its contents. Note that this does not obey +include/exclude filters - everything will be removed. Use delete if you +want to selectively delete files. + + rclone purge remote:path [flags] + +Options + + -h, --help help for purge + + +rclone mkdir + +Make the path if it doesn't already exist. + +Synopsis + +Make the path if it doesn't already exist. + + rclone mkdir remote:path [flags] + +Options + + -h, --help help for mkdir + + +rclone rmdir + +Remove the path if empty. + +Synopsis + +Remove the path. Note that you can't remove a path with objects in it, +use purge for that. + + rclone rmdir remote:path [flags] + +Options + + -h, --help help for rmdir + + +rclone check + +Checks the files in the source and destination match. + +Synopsis + +Checks the files in the source and destination match. It compares sizes +and hashes (MD5 or SHA1) and logs a report of files which don't match. +It doesn't alter the source or destination. + +If you supply the --size-only flag, it will only compare the sizes not +the hashes as well. Use this for a quick check. + +If you supply the --download flag, it will download the data from both +remotes and check them against each other on the fly. This can be useful +for remotes that don't support hashes or if you really want to check all +the data. + +If you supply the --one-way flag, it will only check that files in +source match the files in destination, not the other way around. Meaning +extra files in destination that are not in the source will not trigger +an error. + + rclone check source:path dest:path [flags] + +Options + + --download Check by downloading rather than with hash. + -h, --help help for check + --one-way Check one way only, source files must exist on remote + + +rclone ls + +List the objects in the path with size and path. + +Synopsis + +Lists the objects in the source path to standard output in a human +readable format with size and path. Recurses by default. + +Eg + + $ rclone ls swift:bucket + 60295 bevajer5jef + 90613 canole + 94467 diwogej7 + 37600 fubuwic + +Any of the filtering options can be applied to this commmand. + +There are several related list commands + +- ls to list size and path of objects only +- lsl to list modification time, size and path of objects only +- lsd to list directories only +- lsf to list objects and directories in easy to parse format +- lsjson to list objects and directories in JSON format + +ls,lsl,lsd are designed to be human readable. lsf is designed to be +human and machine readable. lsjson is designed to be machine readable. + +Note that ls and lsl recurse by default - use "--max-depth 1" to stop +the recursion. + +The other list commands lsd,lsf,lsjson do not recurse by default - use +"-R" to make them recurse. + +Listing a non existent directory will produce an error except for +remotes which can't have empty directories (eg s3, swift, gcs, etc - the +bucket based remotes). + + rclone ls remote:path [flags] + +Options + + -h, --help help for ls + + +rclone lsd + +List all directories/containers/buckets in the path. + +Synopsis + +Lists the directories in the source path to standard output. Does not +recurse by default. Use the -R flag to recurse. + +This command lists the total size of the directory (if known, -1 if +not), the modification time (if known, the current time if not), the +number of objects in the directory (if known, -1 if not) and the name of +the directory, Eg + + $ rclone lsd swift: + 494000 2018-04-26 08:43:20 10000 10000files + 65 2018-04-26 08:43:20 1 1File + +Or + + $ rclone lsd drive:test + -1 2016-10-17 17:41:53 -1 1000files + -1 2017-01-03 14:40:54 -1 2500files + -1 2017-07-08 14:39:28 -1 4000files + +If you just want the directory names use "rclone lsf --dirs-only". + +Any of the filtering options can be applied to this commmand. + +There are several related list commands + +- ls to list size and path of objects only +- lsl to list modification time, size and path of objects only +- lsd to list directories only +- lsf to list objects and directories in easy to parse format +- lsjson to list objects and directories in JSON format + +ls,lsl,lsd are designed to be human readable. lsf is designed to be +human and machine readable. lsjson is designed to be machine readable. + +Note that ls and lsl recurse by default - use "--max-depth 1" to stop +the recursion. + +The other list commands lsd,lsf,lsjson do not recurse by default - use +"-R" to make them recurse. + +Listing a non existent directory will produce an error except for +remotes which can't have empty directories (eg s3, swift, gcs, etc - the +bucket based remotes). + + rclone lsd remote:path [flags] + +Options + + -h, --help help for lsd + -R, --recursive Recurse into the listing. + + +rclone lsl + +List the objects in path with modification time, size and path. + +Synopsis + +Lists the objects in the source path to standard output in a human +readable format with modification time, size and path. Recurses by +default. + +Eg + + $ rclone lsl swift:bucket + 60295 2016-06-25 18:55:41.062626927 bevajer5jef + 90613 2016-06-25 18:55:43.302607074 canole + 94467 2016-06-25 18:55:43.046609333 diwogej7 + 37600 2016-06-25 18:55:40.814629136 fubuwic + +Any of the filtering options can be applied to this commmand. + +There are several related list commands + +- ls to list size and path of objects only +- lsl to list modification time, size and path of objects only +- lsd to list directories only +- lsf to list objects and directories in easy to parse format +- lsjson to list objects and directories in JSON format + +ls,lsl,lsd are designed to be human readable. lsf is designed to be +human and machine readable. lsjson is designed to be machine readable. + +Note that ls and lsl recurse by default - use "--max-depth 1" to stop +the recursion. + +The other list commands lsd,lsf,lsjson do not recurse by default - use +"-R" to make them recurse. + +Listing a non existent directory will produce an error except for +remotes which can't have empty directories (eg s3, swift, gcs, etc - the +bucket based remotes). + + rclone lsl remote:path [flags] + +Options + + -h, --help help for lsl + + +rclone md5sum + +Produces an md5sum file for all the objects in the path. + +Synopsis + +Produces an md5sum file for all the objects in the path. This is in the +same format as the standard md5sum tool produces. + + rclone md5sum remote:path [flags] + +Options + + -h, --help help for md5sum + + +rclone sha1sum + +Produces an sha1sum file for all the objects in the path. + +Synopsis + +Produces an sha1sum file for all the objects in the path. This is in the +same format as the standard sha1sum tool produces. + + rclone sha1sum remote:path [flags] + +Options + + -h, --help help for sha1sum + + +rclone size + +Prints the total size and number of objects in remote:path. + +Synopsis + +Prints the total size and number of objects in remote:path. + + rclone size remote:path [flags] + +Options + + -h, --help help for size + --json format output as JSON + + +rclone version + +Show the version number. + +Synopsis + +Show the version number, the go version and the architecture. + +Eg + + $ rclone version + rclone v1.41 + - os/arch: linux/amd64 + - go version: go1.10 + +If you supply the --check flag, then it will do an online check to +compare your version with the latest release and the latest beta. + + $ rclone version --check + yours: 1.42.0.6 + latest: 1.42 (released 2018-06-16) + beta: 1.42.0.5 (released 2018-06-17) + +Or + + $ rclone version --check + yours: 1.41 + latest: 1.42 (released 2018-06-16) + upgrade: https://downloads.rclone.org/v1.42 + beta: 1.42.0.5 (released 2018-06-17) + upgrade: https://beta.rclone.org/v1.42-005-g56e1e820 + + rclone version [flags] + +Options + + --check Check for new version. + -h, --help help for version + + +rclone cleanup + +Clean up the remote if possible + +Synopsis + +Clean up the remote if possible. Empty the trash or delete old file +versions. Not supported by all remotes. + + rclone cleanup remote:path [flags] + +Options + + -h, --help help for cleanup + + +rclone dedupe + +Interactively find duplicate files and delete/rename them. + +Synopsis + +By default dedupe interactively finds duplicate files and offers to +delete all but one or rename them to be different. Only useful with +Google Drive which can have duplicate file names. + +In the first pass it will merge directories with the same name. It will +do this iteratively until all the identical directories have been +merged. + +The dedupe command will delete all but one of any identical (same +md5sum) files it finds without confirmation. This means that for most +duplicated files the dedupe command will not be interactive. You can use +--dry-run to see what would happen without doing anything. + +Here is an example run. + +Before - with duplicates + + $ rclone lsl drive:dupes + 6048320 2016-03-05 16:23:16.798000000 one.txt + 6048320 2016-03-05 16:23:11.775000000 one.txt + 564374 2016-03-05 16:23:06.731000000 one.txt + 6048320 2016-03-05 16:18:26.092000000 one.txt + 6048320 2016-03-05 16:22:46.185000000 two.txt + 1744073 2016-03-05 16:22:38.104000000 two.txt + 564374 2016-03-05 16:22:52.118000000 two.txt + +Now the dedupe session + + $ rclone dedupe drive:dupes + 2016/03/05 16:24:37 Google drive root 'dupes': Looking for duplicates using interactive mode. + one.txt: Found 4 duplicates - deleting identical copies + one.txt: Deleting 2/3 identical duplicates (md5sum "1eedaa9fe86fd4b8632e2ac549403b36") + one.txt: 2 duplicates remain + 1: 6048320 bytes, 2016-03-05 16:23:16.798000000, md5sum 1eedaa9fe86fd4b8632e2ac549403b36 + 2: 564374 bytes, 2016-03-05 16:23:06.731000000, md5sum 7594e7dc9fc28f727c42ee3e0749de81 + s) Skip and do nothing + k) Keep just one (choose which in next step) + r) Rename all to be different (by changing file.jpg to file-1.jpg) + s/k/r> k + Enter the number of the file to keep> 1 + one.txt: Deleted 1 extra copies + two.txt: Found 3 duplicates - deleting identical copies + two.txt: 3 duplicates remain + 1: 564374 bytes, 2016-03-05 16:22:52.118000000, md5sum 7594e7dc9fc28f727c42ee3e0749de81 + 2: 6048320 bytes, 2016-03-05 16:22:46.185000000, md5sum 1eedaa9fe86fd4b8632e2ac549403b36 + 3: 1744073 bytes, 2016-03-05 16:22:38.104000000, md5sum 851957f7fb6f0bc4ce76be966d336802 + s) Skip and do nothing + k) Keep just one (choose which in next step) + r) Rename all to be different (by changing file.jpg to file-1.jpg) + s/k/r> r + two-1.txt: renamed from: two.txt + two-2.txt: renamed from: two.txt + two-3.txt: renamed from: two.txt + +The result being + + $ rclone lsl drive:dupes + 6048320 2016-03-05 16:23:16.798000000 one.txt + 564374 2016-03-05 16:22:52.118000000 two-1.txt + 6048320 2016-03-05 16:22:46.185000000 two-2.txt + 1744073 2016-03-05 16:22:38.104000000 two-3.txt + +Dedupe can be run non interactively using the --dedupe-mode flag or by +using an extra parameter with the same value + +- --dedupe-mode interactive - interactive as above. +- --dedupe-mode skip - removes identical files then skips anything + left. +- --dedupe-mode first - removes identical files then keeps the first + one. +- --dedupe-mode newest - removes identical files then keeps the newest + one. +- --dedupe-mode oldest - removes identical files then keeps the oldest + one. +- --dedupe-mode largest - removes identical files then keeps the + largest one. +- --dedupe-mode rename - removes identical files then renames the rest + to be different. + +For example to rename all the identically named photos in your Google +Photos directory, do + + rclone dedupe --dedupe-mode rename "drive:Google Photos" + +Or + + rclone dedupe rename "drive:Google Photos" + + rclone dedupe [mode] remote:path [flags] + +Options + + --dedupe-mode string Dedupe mode interactive|skip|first|newest|oldest|rename. (default "interactive") + -h, --help help for dedupe + + +rclone about + +Get quota information from the remote. + +Synopsis + +Get quota information from the remote, like bytes used/free/quota and +bytes used in the trash. Not supported by all remotes. + +This will print to stdout something like this: + + Total: 17G + Used: 7.444G + Free: 1.315G + Trashed: 100.000M + Other: 8.241G + +Where the fields are: + +- Total: total size available. +- Used: total size used +- Free: total amount this user could upload. +- Trashed: total amount in the trash +- Other: total amount in other storage (eg Gmail, Google Photos) +- Objects: total number of objects in the storage + +Note that not all the backends provide all the fields - they will be +missing if they are not known for that backend. Where it is known that +the value is unlimited the value will also be omitted. + +Use the --full flag to see the numbers written out in full, eg + + Total: 18253611008 + Used: 7993453766 + Free: 1411001220 + Trashed: 104857602 + Other: 8849156022 + +Use the --json flag for a computer readable output, eg + + { + "total": 18253611008, + "used": 7993453766, + "trashed": 104857602, + "other": 8849156022, + "free": 1411001220 + } + + rclone about remote: [flags] + +Options + + --full Full numbers instead of SI units + -h, --help help for about + --json Format output as JSON + + +rclone authorize + +Remote authorization. + +Synopsis + +Remote authorization. Used to authorize a remote or headless rclone from +a machine with a browser - use as instructed by rclone config. + + rclone authorize [flags] + +Options + + -h, --help help for authorize + + +rclone cachestats + +Print cache stats for a remote + +Synopsis + +Print cache stats for a remote in JSON format + + rclone cachestats source: [flags] + +Options + + -h, --help help for cachestats + + +rclone cat + +Concatenates any files and sends them to stdout. + +Synopsis + +rclone cat sends any files to standard output. + +You can use it like this to output a single file + + rclone cat remote:path/to/file + +Or like this to output any file in dir or subdirectories. + + rclone cat remote:path/to/dir + +Or like this to output any .txt files in dir or subdirectories. + + rclone --include "*.txt" cat remote:path/to/dir + +Use the --head flag to print characters only at the start, --tail for +the end and --offset and --count to print a section in the middle. Note +that if offset is negative it will count from the end, so --offset -1 +--count 1 is equivalent to --tail 1. + + rclone cat remote:path [flags] + +Options + + --count int Only print N characters. (default -1) + --discard Discard the output instead of printing. + --head int Only print the first N characters. + -h, --help help for cat + --offset int Start printing at offset N (or from end if -ve). + --tail int Only print the last N characters. + + +rclone config create + +Create a new remote with name, type and options. + +Synopsis + +Create a new remote of with and options. The options should be passed in +in pairs of . + +For example to make a swift remote of name myremote using auto config +you would do: + + rclone config create myremote swift env_auth true + + rclone config create [ ]* [flags] + +Options + + -h, --help help for create + + +rclone config delete + +Delete an existing remote . + +Synopsis + +Delete an existing remote . + + rclone config delete [flags] + +Options + + -h, --help help for delete + + +rclone config dump + +Dump the config file as JSON. + +Synopsis + +Dump the config file as JSON. + + rclone config dump [flags] + +Options + + -h, --help help for dump + + +rclone config edit + +Enter an interactive configuration session. + +Synopsis + +Enter an interactive configuration session where you can setup new +remotes and manage existing ones. You may also set or remove a password +to protect your configuration. + + rclone config edit [flags] + +Options + + -h, --help help for edit + + +rclone config file + +Show path of configuration file in use. + +Synopsis + +Show path of configuration file in use. + + rclone config file [flags] + +Options + + -h, --help help for file + + +rclone config password + +Update password in an existing remote. + +Synopsis + +Update an existing remote's password. The password should be passed in +in pairs of . + +For example to set password of a remote of name myremote you would do: + + rclone config password myremote fieldname mypassword + + rclone config password [ ]+ [flags] + +Options + + -h, --help help for password + + +rclone config providers + +List in JSON format all the providers and options. + +Synopsis + +List in JSON format all the providers and options. + + rclone config providers [flags] + +Options + + -h, --help help for providers + + +rclone config show + +Print (decrypted) config file, or the config for a single remote. + +Synopsis + +Print (decrypted) config file, or the config for a single remote. + + rclone config show [] [flags] + +Options + + -h, --help help for show + + +rclone config update + +Update options in an existing remote. + +Synopsis + +Update an existing remote's options. The options should be passed in in +pairs of . + +For example to update the env_auth field of a remote of name myremote +you would do: + + rclone config update myremote swift env_auth true + + rclone config update [ ]+ [flags] + +Options + + -h, --help help for update + + +rclone copyto + +Copy files from source to dest, skipping already copied + +Synopsis + +If source:path is a file or directory then it copies it to a file or +directory named dest:path. + +This can be used to upload single files to other than their current +name. If the source is a directory then it acts exactly like the copy +command. + +So + + rclone copyto src dst + +where src and dst are rclone paths, either remote:path or /path/to/local +or C:. + +This will: + + if src is file + copy it to dst, overwriting an existing file if it exists + if src is directory + copy it to dst, overwriting existing files if they exist + see copy command for full details + +This doesn't transfer unchanged files, testing by size and modification +time or MD5SUM. It doesn't delete files from the destination. + + rclone copyto source:path dest:path [flags] + +Options + + -h, --help help for copyto + + +rclone copyurl + +Copy url content to dest. + +Synopsis + +Download urls content and copy it to destination without saving it in +tmp storage. + + rclone copyurl https://example.com dest:path [flags] + +Options + + -h, --help help for copyurl + + +rclone cryptcheck + +Cryptcheck checks the integrity of a crypted remote. + +Synopsis + +rclone cryptcheck checks a remote against a crypted remote. This is the +equivalent of running rclone check, but able to check the checksums of +the crypted remote. + +For it to work the underlying remote of the cryptedremote must support +some kind of checksum. + +It works by reading the nonce from each file on the cryptedremote: and +using that to encrypt each file on the remote:. It then checks the +checksum of the underlying file on the cryptedremote: against the +checksum of the file it has just encrypted. + +Use it like this + + rclone cryptcheck /path/to/files encryptedremote:path + +You can use it like this also, but that will involve downloading all the +files in remote:path. + + rclone cryptcheck remote:path encryptedremote:path + +After it has run it will log the status of the encryptedremote:. + +If you supply the --one-way flag, it will only check that files in +source match the files in destination, not the other way around. Meaning +extra files in destination that are not in the source will not trigger +an error. + + rclone cryptcheck remote:path cryptedremote:path [flags] + +Options + + -h, --help help for cryptcheck + --one-way Check one way only, source files must exist on destination + + +rclone cryptdecode + +Cryptdecode returns unencrypted file names. + +Synopsis + +rclone cryptdecode returns unencrypted file names when provided with a +list of encrypted file names. List limit is 10 items. + +If you supply the --reverse flag, it will return encrypted file names. + +use it like this + + rclone cryptdecode encryptedremote: encryptedfilename1 encryptedfilename2 + + rclone cryptdecode --reverse encryptedremote: filename1 filename2 + + rclone cryptdecode encryptedremote: encryptedfilename [flags] + +Options + + -h, --help help for cryptdecode + --reverse Reverse cryptdecode, encrypts filenames + + +rclone dbhashsum + +Produces a Dropbox hash file for all the objects in the path. + +Synopsis + +Produces a Dropbox hash file for all the objects in the path. The hashes +are calculated according to Dropbox content hash rules. The output is in +the same format as md5sum and sha1sum. + + rclone dbhashsum remote:path [flags] + +Options + + -h, --help help for dbhashsum + + +rclone deletefile + +Remove a single file from remote. + +Synopsis + +Remove a single file from remote. Unlike delete it cannot be used to +remove a directory and it doesn't obey include/exclude filters - if the +specified file exists, it will always be removed. + + rclone deletefile remote:path [flags] + +Options + + -h, --help help for deletefile + + +rclone genautocomplete + +Output completion script for a given shell. + +Synopsis + +Generates a shell completion script for rclone. Run with --help to list +the supported shells. + +Options + + -h, --help help for genautocomplete + + +rclone genautocomplete bash + +Output bash completion script for rclone. + +Synopsis + +Generates a bash shell autocompletion script for rclone. + +This writes to /etc/bash_completion.d/rclone by default so will probably +need to be run with sudo or as root, eg + + sudo rclone genautocomplete bash + +Logout and login again to use the autocompletion scripts, or source them +directly + + . /etc/bash_completion + +If you supply a command line argument the script will be written there. + + rclone genautocomplete bash [output_file] [flags] + +Options + + -h, --help help for bash + + +rclone genautocomplete zsh + +Output zsh completion script for rclone. + +Synopsis + +Generates a zsh autocompletion script for rclone. + +This writes to /usr/share/zsh/vendor-completions/_rclone by default so +will probably need to be run with sudo or as root, eg + + sudo rclone genautocomplete zsh + +Logout and login again to use the autocompletion scripts, or source them +directly + + autoload -U compinit && compinit + +If you supply a command line argument the script will be written there. + + rclone genautocomplete zsh [output_file] [flags] + +Options + + -h, --help help for zsh + + +rclone gendocs + +Output markdown docs for rclone to the directory supplied. + +Synopsis + +This produces markdown docs for the rclone commands to the directory +supplied. These are in a format suitable for hugo to render into the +rclone.org website. + + rclone gendocs output_directory [flags] + +Options + + -h, --help help for gendocs + + +rclone hashsum + +Produces an hashsum file for all the objects in the path. + +Synopsis + +Produces a hash file for all the objects in the path using the hash +named. The output is in the same format as the standard md5sum/sha1sum +tool. + +Run without a hash to see the list of supported hashes, eg + + $ rclone hashsum + Supported hashes are: + * MD5 + * SHA-1 + * DropboxHash + * QuickXorHash + +Then + + $ rclone hashsum MD5 remote:path + + rclone hashsum remote:path [flags] + +Options + + -h, --help help for hashsum + + +rclone link + +Generate public link to file/folder. + +Synopsis + +rclone link will create or retrieve a public link to the given file or +folder. + + rclone link remote:path/to/file + rclone link remote:path/to/folder/ + +If successful, the last line of the output will contain the link. Exact +capabilities depend on the remote, but the link will always be created +with the least constraints – e.g. no expiry, no password protection, +accessible without account. + + rclone link remote:path [flags] + +Options + + -h, --help help for link + + +rclone listremotes + +List all the remotes in the config file. + +Synopsis + +rclone listremotes lists all the available remotes from the config file. + +When uses with the -l flag it lists the types too. + + rclone listremotes [flags] + +Options + + -h, --help help for listremotes + -l, --long Show the type as well as names. + + +rclone lsf + +List directories and objects in remote:path formatted for parsing + +Synopsis + +List the contents of the source path (directories and objects) to +standard output in a form which is easy to parse by scripts. By default +this will just be the names of the objects and directories, one per +line. The directories will have a / suffix. + +Eg + + $ rclone lsf swift:bucket + bevajer5jef + canole + diwogej7 + ferejej3gux/ + fubuwic + +Use the --format option to control what gets listed. By default this is +just the path, but you can use these parameters to control the output: + + p - path + s - size + t - modification time + h - hash + i - ID of object if known + m - MimeType of object if known + +So if you wanted the path, size and modification time, you would use +--format "pst", or maybe --format "tsp" to put the path last. + +Eg + + $ rclone lsf --format "tsp" swift:bucket + 2016-06-25 18:55:41;60295;bevajer5jef + 2016-06-25 18:55:43;90613;canole + 2016-06-25 18:55:43;94467;diwogej7 + 2018-04-26 08:50:45;0;ferejej3gux/ + 2016-06-25 18:55:40;37600;fubuwic + +If you specify "h" in the format you will get the MD5 hash by default, +use the "--hash" flag to change which hash you want. Note that this can +be returned as an empty string if it isn't available on the object (and +for directories), "ERROR" if there was an error reading it from the +object and "UNSUPPORTED" if that object does not support that hash type. + +For example to emulate the md5sum command you can use + + rclone lsf -R --hash MD5 --format hp --separator " " --files-only . + +Eg + + $ rclone lsf -R --hash MD5 --format hp --separator " " --files-only swift:bucket + 7908e352297f0f530b84a756f188baa3 bevajer5jef + cd65ac234e6fea5925974a51cdd865cc canole + 03b5341b4f234b9d984d03ad076bae91 diwogej7 + 8fd37c3810dd660778137ac3a66cc06d fubuwic + 99713e14a4c4ff553acaf1930fad985b gixacuh7ku + +(Though "rclone md5sum ." is an easier way of typing this.) + +By default the separator is ";" this can be changed with the --separator +flag. Note that separators aren't escaped in the path so putting it last +is a good strategy. + +Eg + + $ rclone lsf --separator "," --format "tshp" swift:bucket + 2016-06-25 18:55:41,60295,7908e352297f0f530b84a756f188baa3,bevajer5jef + 2016-06-25 18:55:43,90613,cd65ac234e6fea5925974a51cdd865cc,canole + 2016-06-25 18:55:43,94467,03b5341b4f234b9d984d03ad076bae91,diwogej7 + 2018-04-26 08:52:53,0,,ferejej3gux/ + 2016-06-25 18:55:40,37600,8fd37c3810dd660778137ac3a66cc06d,fubuwic + +You can output in CSV standard format. This will escape things in " if +they contain , + +Eg + + $ rclone lsf --csv --files-only --format ps remote:path + test.log,22355 + test.sh,449 + "this file contains a comma, in the file name.txt",6 + +Note that the --absolute parameter is useful for making lists of files +to pass to an rclone copy with the --files-from flag. + +For example to find all the files modified within one day and copy those +only (without traversing the whole directory structure): + + rclone lsf --absolute --files-only --max-age 1d /path/to/local > new_files + rclone copy --files-from new_files /path/to/local remote:path + +Any of the filtering options can be applied to this commmand. + +There are several related list commands + +- ls to list size and path of objects only +- lsl to list modification time, size and path of objects only +- lsd to list directories only +- lsf to list objects and directories in easy to parse format +- lsjson to list objects and directories in JSON format + +ls,lsl,lsd are designed to be human readable. lsf is designed to be +human and machine readable. lsjson is designed to be machine readable. + +Note that ls and lsl recurse by default - use "--max-depth 1" to stop +the recursion. + +The other list commands lsd,lsf,lsjson do not recurse by default - use +"-R" to make them recurse. + +Listing a non existent directory will produce an error except for +remotes which can't have empty directories (eg s3, swift, gcs, etc - the +bucket based remotes). + + rclone lsf remote:path [flags] + +Options + + --absolute Put a leading / in front of path names. + --csv Output in CSV format. + -d, --dir-slash Append a slash to directory names. (default true) + --dirs-only Only list directories. + --files-only Only list files. + -F, --format string Output format - see help for details (default "p") + --hash h Use this hash when h is used in the format MD5|SHA-1|DropboxHash (default "MD5") + -h, --help help for lsf + -R, --recursive Recurse into the listing. + -s, --separator string Separator for the items in the format. (default ";") + + +rclone lsjson + +List directories and objects in the path in JSON format. + +Synopsis + +List directories and objects in the path in JSON format. + +The output is an array of Items, where each Item looks like this + +{ "Hashes" : { "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f", +"MD5" : "b1946ac92492d2347c6235b4d2611184", "DropboxHash" : +"ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc" }, +"ID": "y2djkhiujf83u33", "OrigID": "UYOJVTUW00Q1RzTDA", "IsDir" : false, +"MimeType" : "application/octet-stream", "ModTime" : +"2017-05-31T16:15:57.034468261+01:00", "Name" : "file.txt", "Encrypted" +: "v0qpsdq8anpci8n929v3uu9338", "Path" : "full/path/goes/here/file.txt", +"Size" : 6 } + +If --hash is not specified the Hashes property won't be emitted. + +If --no-modtime is specified then ModTime will be blank. + +If --encrypted is not specified the Encrypted won't be emitted. + +The Path field will only show folders below the remote path being +listed. If "remote:path" contains the file "subfolder/file.txt", the +Path for "file.txt" will be "subfolder/file.txt", not +"remote:path/subfolder/file.txt". When used without --recursive the Path +will always be the same as Name. + +The time is in RFC3339 format with nanosecond precision. + +The whole output can be processed as a JSON blob, or alternatively it +can be processed line by line as each item is written one to a line. + +Any of the filtering options can be applied to this commmand. + +There are several related list commands + +- ls to list size and path of objects only +- lsl to list modification time, size and path of objects only +- lsd to list directories only +- lsf to list objects and directories in easy to parse format +- lsjson to list objects and directories in JSON format + +ls,lsl,lsd are designed to be human readable. lsf is designed to be +human and machine readable. lsjson is designed to be machine readable. + +Note that ls and lsl recurse by default - use "--max-depth 1" to stop +the recursion. + +The other list commands lsd,lsf,lsjson do not recurse by default - use +"-R" to make them recurse. + +Listing a non existent directory will produce an error except for +remotes which can't have empty directories (eg s3, swift, gcs, etc - the +bucket based remotes). + + rclone lsjson remote:path [flags] + +Options + + -M, --encrypted Show the encrypted names. + --hash Include hashes in the output (may take longer). + -h, --help help for lsjson + --no-modtime Don't read the modification time (can speed things up). + --original Show the ID of the underlying Object. + -R, --recursive Recurse into the listing. + + +rclone mount + +Mount the remote as a mountpoint. EXPERIMENTAL + +Synopsis + +rclone mount allows Linux, FreeBSD, macOS and Windows to mount any of +Rclone's cloud storage systems as a file system with FUSE. + +This is EXPERIMENTAL - use with care. + +First set up your remote using rclone config. Check it works with +rclone ls etc. + +Start the mount like this + + rclone mount remote:path/to/files /path/to/local/mount + +Or on Windows like this where X: is an unused drive letter + + rclone mount remote:path/to/files X: + +When the program ends, either via Ctrl+C or receiving a SIGINT or +SIGTERM signal, the mount is automatically stopped. + +The umount operation can fail, for example when the mountpoint is busy. +When that happens, it is the user's responsibility to stop the mount +manually with + + # Linux + fusermount -u /path/to/local/mount + # OS X + umount /path/to/local/mount + +Installing on Windows + +To run rclone mount on Windows, you will need to download and install +WinFsp. + +WinFsp is an open source Windows File System Proxy which makes it easy +to write user space file systems for Windows. It provides a FUSE +emulation layer which rclone uses combination with cgofuse. Both of +these packages are by Bill Zissimopoulos who was very helpful during the +implementation of rclone mount for Windows. + +Windows caveats + +Note that drives created as Administrator are not visible by other +accounts (including the account that was elevated as Administrator). So +if you start a Windows drive from an Administrative Command Prompt and +then try to access the same drive from Explorer (which does not run as +Administrator), you will not be able to see the new drive. + +The easiest way around this is to start the drive from a normal command +prompt. It is also possible to start a drive from the SYSTEM account +(using the WinFsp.Launcher infrastructure) which creates drives +accessible for everyone on the system or alternatively using the nssm +service manager. + +Limitations + +Without the use of "--vfs-cache-mode" this can only write files +sequentially, it can only seek when reading. This means that many +applications won't work with their files on an rclone mount without +"--vfs-cache-mode writes" or "--vfs-cache-mode full". See the File +Caching section for more info. + +The bucket based remotes (eg Swift, S3, Google Compute Storage, B2, +Hubic) won't work from the root - you will need to specify a bucket, or +a path within the bucket. So swift: won't work whereas swift:bucket will +as will swift:bucket/path. None of these support the concept of +directories, so empty directories will have a tendency to disappear once +they fall out of the directory cache. + +Only supported on Linux, FreeBSD, OS X and Windows at the moment. + +rclone mount vs rclone sync/copy + +File systems expect things to be 100% reliable, whereas cloud storage +systems are a long way from 100% reliable. The rclone sync/copy commands +cope with this with lots of retries. However rclone mount can't use +retries in the same way without making local copies of the uploads. Look +at the EXPERIMENTAL file caching for solutions to make mount mount more +reliable. + +Attribute caching + +You can use the flag --attr-timeout to set the time the kernel caches +the attributes (size, modification time etc) for directory entries. + +The default is "1s" which caches files just long enough to avoid too +many callbacks to rclone from the kernel. + +In theory 0s should be the correct value for filesystems which can +change outside the control of the kernel. However this causes quite a +few problems such as rclone using too much memory, rclone not serving +files to samba and excessive time listing directories. + +The kernel can cache the info about a file for the time given by +"--attr-timeout". You may see corruption if the remote file changes +length during this window. It will show up as either a truncated file or +a file with garbage on the end. With "--attr-timeout 1s" this is very +unlikely but not impossible. The higher you set "--attr-timeout" the +more likely it is. The default setting of "1s" is the lowest setting +which mitigates the problems above. + +If you set it higher ('10s' or '1m' say) then the kernel will call back +to rclone less often making it more efficient, however there is more +chance of the corruption issue above. + +If files don't change on the remote outside of the control of rclone +then there is no chance of corruption. + +This is the same as setting the attr_timeout option in mount.fuse. + +Filters + +Note that all the rclone filters can be used to select a subset of the +files to be visible in the mount. + +systemd + +When running rclone mount as a systemd service, it is possible to use +Type=notify. In this case the service will enter the started state after +the mountpoint has been successfully set up. Units having the rclone +mount service specified as a requirement will see all files and folders +immediately in this mode. + +chunked reading + +--vfs-read-chunk-size will enable reading the source objects in parts. +This can reduce the used download quota for some remotes by requesting +only chunks from the remote that are actually read at the cost of an +increased number of requests. + +When --vfs-read-chunk-size-limit is also specified and greater than +--vfs-read-chunk-size, the chunk size for each open file will get +doubled for each chunk read, until the specified value is reached. A +value of -1 will disable the limit and the chunk size will grow +indefinitely. + +With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the +following parts will be downloaded: 0-100M, 100M-200M, 200M-300M, +300M-400M and so on. When --vfs-read-chunk-size-limit 500M is specified, +the result would be 0-100M, 100M-300M, 300M-700M, 700M-1200M, +1200M-1700M and so on. + +Chunked reading will only work with --vfs-cache-mode < full, as the file +will always be copied to the vfs cache before opening with +--vfs-cache-mode full. + +Directory Cache + +Using the --dir-cache-time flag, you can set how long a directory should +be considered up to date and not refreshed from the backend. Changes +made locally in the mount may appear immediately or invalidate the +cache. However, changes done on the remote will only be picked up once +the cache expires. + +Alternatively, you can send a SIGHUP signal to rclone for it to flush +all directory caches, regardless of how old they are. Assuming only one +rclone instance is running, you can reset the cache like this: + + kill -SIGHUP $(pidof rclone) + +If you configure rclone with a remote control then you can use rclone rc +to flush the whole directory cache: + + rclone rc vfs/forget + +Or individual files or directories: + + rclone rc vfs/forget file=path/to/file dir=path/to/dir + +File Buffering + +The --buffer-size flag determines the amount of memory, that will be +used to buffer data in advance. + +Each open file descriptor will try to keep the specified amount of data +in memory at all times. The buffered data is bound to one file +descriptor and won't be shared between multiple open file descriptors of +the same file. + +This flag is a upper limit for the used memory per file descriptor. The +buffer will only use memory for data that is downloaded but not not yet +read. If the buffer is empty, only a small amount of memory will be +used. The maximum memory used by rclone for buffering can be up to +--buffer-size * open files. + +File Caching + +NB File caching is EXPERIMENTAL - use with care! + +These flags control the VFS file caching options. The VFS layer is used +by rclone mount to make a cloud storage system work more like a normal +file system. + +You'll need to enable VFS caching if you want, for example, to read and +write simultaneously to a file. See below for more details. + +Note that the VFS cache works in addition to the cache backend and you +may find that you need one or the other or both. + + --cache-dir string Directory rclone will use for caching. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + +If run with -vv rclone will print the location of the file cache. The +files are stored in the user cache file area which is OS dependent but +can be controlled with --cache-dir or setting the appropriate +environment variable. + +The cache has 4 different modes selected by --vfs-cache-mode. The higher +the cache mode the more compatible rclone becomes at the cost of using +disk space. + +Note that files are written back to the remote only when they are closed +so if rclone is quit or dies with open files then these won't get +written back to the remote. However they will still be in the on disk +cache. + +--vfs-cache-mode off + +In this mode the cache will read directly from the remote and write +directly to the remote without caching anything on disk. + +This will mean some operations are not possible + +- Files can't be opened for both read AND write +- Files opened for write can't be seeked +- Existing files opened for write must have O_TRUNC set +- Files open for read with O_TRUNC will be opened write only +- Files open for write only will behave as if O_TRUNC was supplied +- Open modes O_APPEND, O_TRUNC are ignored +- If an upload fails it can't be retried + +--vfs-cache-mode minimal + +This is very similar to "off" except that files opened for read AND +write will be buffered to disks. This means that files opened for write +will be a lot more compatible, but uses the minimal disk space. + +These operations are not possible + +- Files opened for write only can't be seeked +- Existing files opened for write must have O_TRUNC set +- Files opened for write only will ignore O_APPEND, O_TRUNC +- If an upload fails it can't be retried + +--vfs-cache-mode writes + +In this mode files opened for read only are still read directly from the +remote, write only and read/write files are buffered to disk first. + +This mode should support all normal file system operations. + +If an upload fails it will be retried up to --low-level-retries times. + +--vfs-cache-mode full + +In this mode all reads and writes are buffered to and from disk. When a +file is opened for read it will be downloaded in its entirety first. + +This may be appropriate for your needs, or you may prefer to look at the +cache backend which does a much more sophisticated job of caching, +including caching directory hierarchies and chunks of files. + +In this mode, unlike the others, when a file is written to the disk, it +will be kept on the disk after it is written to the remote. It will be +purged on a schedule according to --vfs-cache-max-age. + +This mode should support all normal file system operations. + +If an upload or download fails it will be retried up to +--low-level-retries times. + + rclone mount remote:path /path/to/mountpoint [flags] + +Options + + --allow-non-empty Allow mounting over a non-empty directory. + --allow-other Allow access to other users. + --allow-root Allow access to root user. + --attr-timeout duration Time for which file/directory attributes are cached. (default 1s) + --daemon Run mount as a daemon (background mode). + --daemon-timeout duration Time limit for rclone to respond to kernel (not supported by all OSes). + --debug-fuse Debug the FUSE internals - needs -v. + --default-permissions Makes kernel enforce access control based on the file mode. + --dir-cache-time duration Time to cache directory entries for. (default 5m0s) + --fuse-flag stringArray Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required. + --gid uint32 Override the gid field set by the filesystem. (default 502) + -h, --help help for mount + --max-read-ahead int The number of bytes that can be prefetched for sequential reads. (default 128k) + --no-checksum Don't compare checksums on up/download. + --no-modtime Don't read/write the modification time (can speed things up). + --no-seek Don't allow seeking in files. + -o, --option stringArray Option for libfuse/WinFsp. Repeat if required. + --poll-interval duration Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable. (default 1m0s) + --read-only Mount read-only. + --uid uint32 Override the uid field set by the filesystem. (default 502) + --umask int Override the permission bits set by the filesystem. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + --vfs-read-chunk-size int Read the source objects in chunks. (default 128M) + --vfs-read-chunk-size-limit int If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off) + --volname string Set the volume name (not supported by all OSes). + --write-back-cache Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used. + + +rclone moveto + +Move file or directory from source to dest. + +Synopsis + +If source:path is a file or directory then it moves it to a file or +directory named dest:path. + +This can be used to rename files or upload single files to other than +their existing name. If the source is a directory then it acts exacty +like the move command. + +So + + rclone moveto src dst + +where src and dst are rclone paths, either remote:path or /path/to/local +or C:. + +This will: + + if src is file + move it to dst, overwriting an existing file if it exists + if src is directory + move it to dst, overwriting existing files if they exist + see move command for full details + +This doesn't transfer unchanged files, testing by size and modification +time or MD5SUM. src will be deleted on successful transfer. + +IMPORTANT: Since this can cause data loss, test first with the --dry-run +flag. + + rclone moveto source:path dest:path [flags] + +Options + + -h, --help help for moveto + + +rclone ncdu + +Explore a remote with a text based user interface. + +Synopsis + +This displays a text based user interface allowing the navigation of a +remote. It is most useful for answering the question - "What is using +all my disk space?". + +To make the user interface it first scans the entire remote given and +builds an in memory representation. rclone ncdu can be used during this +scanning phase and you will see it building up the directory structure +as it goes along. + +Here are the keys - press '?' to toggle the help on and off + + ↑,↓ or k,j to Move + →,l to enter + ←,h to return + c toggle counts + g toggle graph + n,s,C sort by name,size,count + ^L refresh screen + ? to toggle help on and off + q/ESC/c-C to quit + +This an homage to the ncdu tool but for rclone remotes. It is missing +lots of features at the moment, most importantly deleting files, but is +useful as it stands. + + rclone ncdu remote:path [flags] + +Options + + -h, --help help for ncdu + + +rclone obscure + +Obscure password for use in the rclone.conf + +Synopsis + +Obscure password for use in the rclone.conf + + rclone obscure password [flags] + +Options + + -h, --help help for obscure + + +rclone rc + +Run a command against a running rclone. + +Synopsis + +This runs a command against a running rclone. By default it will use +that specified in the --rc-addr command. + +Arguments should be passed in as parameter=value. + +The result will be returned as a JSON object by default. + +Use "rclone rc" to see a list of all possible commands. + + rclone rc commands parameter [flags] + +Options + + -h, --help help for rc + --no-output If set don't output the JSON result. + --url string URL to connect to rclone remote control. (default "http://localhost:5572/") + + +rclone rcat + +Copies standard input to file on remote. + +Synopsis + +rclone rcat reads from standard input (stdin) and copies it to a single +remote file. + + echo "hello world" | rclone rcat remote:path/to/file + ffmpeg - | rclone rcat remote:path/to/file + +If the remote file already exists, it will be overwritten. + +rcat will try to upload small files in a single request, which is +usually more efficient than the streaming/chunked upload endpoints, +which use multiple requests. Exact behaviour depends on the remote. What +is considered a small file may be set through --streaming-upload-cutoff. +Uploading only starts after the cutoff is reached or if the file ends +before that. The data must fit into RAM. The cutoff needs to be small +enough to adhere the limits of your remote, please see there. Generally +speaking, setting this cutoff too high will decrease your performance. + +Note that the upload can also not be retried because the data is not +kept around until the upload succeeds. If you need to transfer a lot of +data, you're better off caching locally and then rclone move it to the +destination. + + rclone rcat remote:path [flags] + +Options + + -h, --help help for rcat + + +rclone rmdirs + +Remove empty directories under the path. + +Synopsis + +This removes any empty directories (or directories that only contain +empty directories) under the path that it finds, including the path if +it has nothing in. + +If you supply the --leave-root flag, it will not remove the root +directory. + +This is useful for tidying up remotes that rclone has left a lot of +empty directories in. + + rclone rmdirs remote:path [flags] + +Options + + -h, --help help for rmdirs + --leave-root Do not remove root directory if empty + + +rclone serve + +Serve a remote over a protocol. + +Synopsis + +rclone serve is used to serve a remote over a given protocol. This +command requires the use of a subcommand to specify the protocol, eg + + rclone serve http remote: + +Each subcommand has its own options which you can see in their help. + + rclone serve [opts] [flags] + +Options + + -h, --help help for serve + + +rclone serve http + +Serve the remote over HTTP. + +Synopsis + +rclone serve http implements a basic web server to serve the remote over +HTTP. This can be viewed in a web browser or you can make a remote of +type http read from it. + +You can use the filter flags (eg --include, --exclude) to control what +is served. + +The server will log errors. Use -v to see access logs. + +--bwlimit will be respected for file transfers. Use --stats to control +the stats printing. + +Server options + +Use --addr to specify which IP address and port the server should listen +on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By +default it only listens on localhost. You can use port :0 to let the OS +choose an available port. + +If you set --addr to listen on a public or LAN accessible IP address +then using Authentication is advised - see the next section for info. + +--server-read-timeout and --server-write-timeout can be used to control +the timeouts on the server. Note that this is the total time for a +transfer. + +--max-header-bytes controls the maximum number of bytes the server will +accept in the HTTP header. + +Authentication + +By default this will serve files without needing a login. + +You can either use an htpasswd file which can take lots of users, or set +a single username and password with the --user and --pass flags. + +Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in +standard apache format and supports MD5, SHA1 and BCrypt for basic +authentication. Bcrypt is recommended. + +To create an htpasswd file: + + touch htpasswd + htpasswd -B htpasswd user + htpasswd -B htpasswd anotherUser + +The password file can be updated while rclone is running. + +Use --realm to set the authentication realm. + +SSL/TLS + +By default this will serve over http. If you want you can serve over +https. You will need to supply the --cert and --key flags. If you wish +to do client side certificate validation then you will need to supply +--client-ca also. + +--cert should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. --key should be the PEM encoded private +key and --client-ca should be the PEM encoded client certificate +authority certificate. + +Directory Cache + +Using the --dir-cache-time flag, you can set how long a directory should +be considered up to date and not refreshed from the backend. Changes +made locally in the mount may appear immediately or invalidate the +cache. However, changes done on the remote will only be picked up once +the cache expires. + +Alternatively, you can send a SIGHUP signal to rclone for it to flush +all directory caches, regardless of how old they are. Assuming only one +rclone instance is running, you can reset the cache like this: + + kill -SIGHUP $(pidof rclone) + +If you configure rclone with a remote control then you can use rclone rc +to flush the whole directory cache: + + rclone rc vfs/forget + +Or individual files or directories: + + rclone rc vfs/forget file=path/to/file dir=path/to/dir + +File Buffering + +The --buffer-size flag determines the amount of memory, that will be +used to buffer data in advance. + +Each open file descriptor will try to keep the specified amount of data +in memory at all times. The buffered data is bound to one file +descriptor and won't be shared between multiple open file descriptors of +the same file. + +This flag is a upper limit for the used memory per file descriptor. The +buffer will only use memory for data that is downloaded but not not yet +read. If the buffer is empty, only a small amount of memory will be +used. The maximum memory used by rclone for buffering can be up to +--buffer-size * open files. + +File Caching + +NB File caching is EXPERIMENTAL - use with care! + +These flags control the VFS file caching options. The VFS layer is used +by rclone mount to make a cloud storage system work more like a normal +file system. + +You'll need to enable VFS caching if you want, for example, to read and +write simultaneously to a file. See below for more details. + +Note that the VFS cache works in addition to the cache backend and you +may find that you need one or the other or both. + + --cache-dir string Directory rclone will use for caching. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + +If run with -vv rclone will print the location of the file cache. The +files are stored in the user cache file area which is OS dependent but +can be controlled with --cache-dir or setting the appropriate +environment variable. + +The cache has 4 different modes selected by --vfs-cache-mode. The higher +the cache mode the more compatible rclone becomes at the cost of using +disk space. + +Note that files are written back to the remote only when they are closed +so if rclone is quit or dies with open files then these won't get +written back to the remote. However they will still be in the on disk +cache. + +--vfs-cache-mode off + +In this mode the cache will read directly from the remote and write +directly to the remote without caching anything on disk. + +This will mean some operations are not possible + +- Files can't be opened for both read AND write +- Files opened for write can't be seeked +- Existing files opened for write must have O_TRUNC set +- Files open for read with O_TRUNC will be opened write only +- Files open for write only will behave as if O_TRUNC was supplied +- Open modes O_APPEND, O_TRUNC are ignored +- If an upload fails it can't be retried + +--vfs-cache-mode minimal + +This is very similar to "off" except that files opened for read AND +write will be buffered to disks. This means that files opened for write +will be a lot more compatible, but uses the minimal disk space. + +These operations are not possible + +- Files opened for write only can't be seeked +- Existing files opened for write must have O_TRUNC set +- Files opened for write only will ignore O_APPEND, O_TRUNC +- If an upload fails it can't be retried + +--vfs-cache-mode writes + +In this mode files opened for read only are still read directly from the +remote, write only and read/write files are buffered to disk first. + +This mode should support all normal file system operations. + +If an upload fails it will be retried up to --low-level-retries times. + +--vfs-cache-mode full + +In this mode all reads and writes are buffered to and from disk. When a +file is opened for read it will be downloaded in its entirety first. + +This may be appropriate for your needs, or you may prefer to look at the +cache backend which does a much more sophisticated job of caching, +including caching directory hierarchies and chunks of files. + +In this mode, unlike the others, when a file is written to the disk, it +will be kept on the disk after it is written to the remote. It will be +purged on a schedule according to --vfs-cache-max-age. + +This mode should support all normal file system operations. + +If an upload or download fails it will be retried up to +--low-level-retries times. + + rclone serve http remote:path [flags] + +Options + + --addr string IPaddress:Port or :Port to bind server to. (default "localhost:8080") + --cert string SSL PEM key (concatenation of certificate and CA certificate) + --client-ca string Client certificate authority to verify clients with + --dir-cache-time duration Time to cache directory entries for. (default 5m0s) + --gid uint32 Override the gid field set by the filesystem. (default 502) + -h, --help help for http + --htpasswd string htpasswd file - if not provided no authentication is done + --key string SSL PEM Private key + --max-header-bytes int Maximum size of request header (default 4096) + --no-checksum Don't compare checksums on up/download. + --no-modtime Don't read/write the modification time (can speed things up). + --no-seek Don't allow seeking in files. + --pass string Password for authentication. + --poll-interval duration Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable. (default 1m0s) + --read-only Mount read-only. + --realm string realm for authentication (default "rclone") + --server-read-timeout duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --uid uint32 Override the uid field set by the filesystem. (default 502) + --umask int Override the permission bits set by the filesystem. (default 2) + --user string User name for authentication. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + --vfs-read-chunk-size int Read the source objects in chunks. (default 128M) + --vfs-read-chunk-size-limit int If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off) + + +rclone serve restic + +Serve the remote for restic's REST API. + +Synopsis + +rclone serve restic implements restic's REST backend API over HTTP. This +allows restic to use rclone as a data storage mechanism for cloud +providers that restic does not support directly. + +Restic is a command line program for doing backups. + +The server will log errors. Use -v to see access logs. + +--bwlimit will be respected for file transfers. Use --stats to control +the stats printing. + +Setting up rclone for use by restic + +First set up a remote for your chosen cloud provider. + +Once you have set up the remote, check it is working with, for example +"rclone lsd remote:". You may have called the remote something other +than "remote:" - just substitute whatever you called it in the following +instructions. + +Now start the rclone restic server + + rclone serve restic -v remote:backup + +Where you can replace "backup" in the above by whatever path in the +remote you wish to use. + +By default this will serve on "localhost:8080" you can change this with +use of the "--addr" flag. + +You might wish to start this server on boot. + +Setting up restic to use rclone + +Now you can follow the restic instructions on setting up restic. + +Note that you will need restic 0.8.2 or later to interoperate with +rclone. + +For the example above you will want to use "http://localhost:8080/" as +the URL for the REST server. + +For example: + + $ export RESTIC_REPOSITORY=rest:http://localhost:8080/ + $ export RESTIC_PASSWORD=yourpassword + $ restic init + created restic backend 8b1a4b56ae at rest:http://localhost:8080/ + + Please note that knowledge of your password is required to access + the repository. Losing your password means that your data is + irrecoverably lost. + $ restic backup /path/to/files/to/backup + scan [/path/to/files/to/backup] + scanned 189 directories, 312 files in 0:00 + [0:00] 100.00% 38.128 MiB / 38.128 MiB 501 / 501 items 0 errors ETA 0:00 + duration: 0:00 + snapshot 45c8fdd8 saved + +Multiple repositories + +Note that you can use the endpoint to host multiple repositories. Do +this by adding a directory name or path after the URL. Note that these +MUST end with /. Eg + + $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user1repo/ + # backup user1 stuff + $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user2repo/ + # backup user2 stuff + +Server options + +Use --addr to specify which IP address and port the server should listen +on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By +default it only listens on localhost. You can use port :0 to let the OS +choose an available port. + +If you set --addr to listen on a public or LAN accessible IP address +then using Authentication is advised - see the next section for info. + +--server-read-timeout and --server-write-timeout can be used to control +the timeouts on the server. Note that this is the total time for a +transfer. + +--max-header-bytes controls the maximum number of bytes the server will +accept in the HTTP header. + +Authentication + +By default this will serve files without needing a login. + +You can either use an htpasswd file which can take lots of users, or set +a single username and password with the --user and --pass flags. + +Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in +standard apache format and supports MD5, SHA1 and BCrypt for basic +authentication. Bcrypt is recommended. + +To create an htpasswd file: + + touch htpasswd + htpasswd -B htpasswd user + htpasswd -B htpasswd anotherUser + +The password file can be updated while rclone is running. + +Use --realm to set the authentication realm. + +SSL/TLS + +By default this will serve over http. If you want you can serve over +https. You will need to supply the --cert and --key flags. If you wish +to do client side certificate validation then you will need to supply +--client-ca also. + +--cert should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. --key should be the PEM encoded private +key and --client-ca should be the PEM encoded client certificate +authority certificate. + + rclone serve restic remote:path [flags] + +Options + + --addr string IPaddress:Port or :Port to bind server to. (default "localhost:8080") + --append-only disallow deletion of repository data + --cert string SSL PEM key (concatenation of certificate and CA certificate) + --client-ca string Client certificate authority to verify clients with + -h, --help help for restic + --htpasswd string htpasswd file - if not provided no authentication is done + --key string SSL PEM Private key + --max-header-bytes int Maximum size of request header (default 4096) + --pass string Password for authentication. + --realm string realm for authentication (default "rclone") + --server-read-timeout duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --stdio run an HTTP2 server on stdin/stdout + --user string User name for authentication. + + +rclone serve webdav + +Serve remote:path over webdav. + +Synopsis + +rclone serve webdav implements a basic webdav server to serve the remote +over HTTP via the webdav protocol. This can be viewed with a webdav +client or you can make a remote of type webdav to read and write it. + +Webdav options + +--etag-hash + +This controls the ETag header. Without this flag the ETag will be based +on the ModTime and Size of the object. + +If this flag is set to "auto" then rclone will choose the first +supported hash on the backend or you can use a named hash such as "MD5" +or "SHA-1". + +Use "rclone hashsum" to see the full list. + +Server options + +Use --addr to specify which IP address and port the server should listen +on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By +default it only listens on localhost. You can use port :0 to let the OS +choose an available port. + +If you set --addr to listen on a public or LAN accessible IP address +then using Authentication is advised - see the next section for info. + +--server-read-timeout and --server-write-timeout can be used to control +the timeouts on the server. Note that this is the total time for a +transfer. + +--max-header-bytes controls the maximum number of bytes the server will +accept in the HTTP header. + +Authentication + +By default this will serve files without needing a login. + +You can either use an htpasswd file which can take lots of users, or set +a single username and password with the --user and --pass flags. + +Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in +standard apache format and supports MD5, SHA1 and BCrypt for basic +authentication. Bcrypt is recommended. + +To create an htpasswd file: + + touch htpasswd + htpasswd -B htpasswd user + htpasswd -B htpasswd anotherUser + +The password file can be updated while rclone is running. + +Use --realm to set the authentication realm. + +SSL/TLS + +By default this will serve over http. If you want you can serve over +https. You will need to supply the --cert and --key flags. If you wish +to do client side certificate validation then you will need to supply +--client-ca also. + +--cert should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. --key should be the PEM encoded private +key and --client-ca should be the PEM encoded client certificate +authority certificate. + +Directory Cache + +Using the --dir-cache-time flag, you can set how long a directory should +be considered up to date and not refreshed from the backend. Changes +made locally in the mount may appear immediately or invalidate the +cache. However, changes done on the remote will only be picked up once +the cache expires. + +Alternatively, you can send a SIGHUP signal to rclone for it to flush +all directory caches, regardless of how old they are. Assuming only one +rclone instance is running, you can reset the cache like this: + + kill -SIGHUP $(pidof rclone) + +If you configure rclone with a remote control then you can use rclone rc +to flush the whole directory cache: + + rclone rc vfs/forget + +Or individual files or directories: + + rclone rc vfs/forget file=path/to/file dir=path/to/dir + +File Buffering + +The --buffer-size flag determines the amount of memory, that will be +used to buffer data in advance. + +Each open file descriptor will try to keep the specified amount of data +in memory at all times. The buffered data is bound to one file +descriptor and won't be shared between multiple open file descriptors of +the same file. + +This flag is a upper limit for the used memory per file descriptor. The +buffer will only use memory for data that is downloaded but not not yet +read. If the buffer is empty, only a small amount of memory will be +used. The maximum memory used by rclone for buffering can be up to +--buffer-size * open files. + +File Caching + +NB File caching is EXPERIMENTAL - use with care! + +These flags control the VFS file caching options. The VFS layer is used +by rclone mount to make a cloud storage system work more like a normal +file system. + +You'll need to enable VFS caching if you want, for example, to read and +write simultaneously to a file. See below for more details. + +Note that the VFS cache works in addition to the cache backend and you +may find that you need one or the other or both. + + --cache-dir string Directory rclone will use for caching. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + +If run with -vv rclone will print the location of the file cache. The +files are stored in the user cache file area which is OS dependent but +can be controlled with --cache-dir or setting the appropriate +environment variable. + +The cache has 4 different modes selected by --vfs-cache-mode. The higher +the cache mode the more compatible rclone becomes at the cost of using +disk space. + +Note that files are written back to the remote only when they are closed +so if rclone is quit or dies with open files then these won't get +written back to the remote. However they will still be in the on disk +cache. + +--vfs-cache-mode off + +In this mode the cache will read directly from the remote and write +directly to the remote without caching anything on disk. + +This will mean some operations are not possible + +- Files can't be opened for both read AND write +- Files opened for write can't be seeked +- Existing files opened for write must have O_TRUNC set +- Files open for read with O_TRUNC will be opened write only +- Files open for write only will behave as if O_TRUNC was supplied +- Open modes O_APPEND, O_TRUNC are ignored +- If an upload fails it can't be retried + +--vfs-cache-mode minimal + +This is very similar to "off" except that files opened for read AND +write will be buffered to disks. This means that files opened for write +will be a lot more compatible, but uses the minimal disk space. + +These operations are not possible + +- Files opened for write only can't be seeked +- Existing files opened for write must have O_TRUNC set +- Files opened for write only will ignore O_APPEND, O_TRUNC +- If an upload fails it can't be retried + +--vfs-cache-mode writes + +In this mode files opened for read only are still read directly from the +remote, write only and read/write files are buffered to disk first. + +This mode should support all normal file system operations. + +If an upload fails it will be retried up to --low-level-retries times. + +--vfs-cache-mode full + +In this mode all reads and writes are buffered to and from disk. When a +file is opened for read it will be downloaded in its entirety first. + +This may be appropriate for your needs, or you may prefer to look at the +cache backend which does a much more sophisticated job of caching, +including caching directory hierarchies and chunks of files. + +In this mode, unlike the others, when a file is written to the disk, it +will be kept on the disk after it is written to the remote. It will be +purged on a schedule according to --vfs-cache-max-age. + +This mode should support all normal file system operations. + +If an upload or download fails it will be retried up to +--low-level-retries times. + + rclone serve webdav remote:path [flags] + +Options + + --addr string IPaddress:Port or :Port to bind server to. (default "localhost:8080") + --cert string SSL PEM key (concatenation of certificate and CA certificate) + --client-ca string Client certificate authority to verify clients with + --dir-cache-time duration Time to cache directory entries for. (default 5m0s) + --etag-hash string Which hash to use for the ETag, or auto or blank for off + --gid uint32 Override the gid field set by the filesystem. (default 502) + -h, --help help for webdav + --htpasswd string htpasswd file - if not provided no authentication is done + --key string SSL PEM Private key + --max-header-bytes int Maximum size of request header (default 4096) + --no-checksum Don't compare checksums on up/download. + --no-modtime Don't read/write the modification time (can speed things up). + --no-seek Don't allow seeking in files. + --pass string Password for authentication. + --poll-interval duration Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable. (default 1m0s) + --read-only Mount read-only. + --realm string realm for authentication (default "rclone") + --server-read-timeout duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --uid uint32 Override the uid field set by the filesystem. (default 502) + --umask int Override the permission bits set by the filesystem. (default 2) + --user string User name for authentication. + --vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s) + --vfs-cache-mode string Cache mode off|minimal|writes|full (default "off") + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s) + --vfs-read-chunk-size int Read the source objects in chunks. (default 128M) + --vfs-read-chunk-size-limit int If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off) + + +rclone touch + +Create new file or change file modification time. + +Synopsis + +Create new file or change file modification time. + + rclone touch remote:path [flags] + +Options + + -h, --help help for touch + -C, --no-create Do not create the file if it does not exist. + -t, --timestamp string Change the modification times to the specified time instead of the current time of day. The argument is of the form 'YYMMDD' (ex. 17.10.30) or 'YYYY-MM-DDTHH:MM:SS' (ex. 2006-01-02T15:04:05) + + +rclone tree + +List the contents of the remote in a tree like fashion. + +Synopsis + +rclone tree lists the contents of a remote in a similar way to the unix +tree command. + +For example + + $ rclone tree remote:path + / + ├── file1 + ├── file2 + ├── file3 + └── subdir + ├── file4 + └── file5 + + 1 directories, 5 files + +You can use any of the filtering options with the tree command (eg +--include and --exclude). You can also use --fast-list. + +The tree command has many options for controlling the listing which are +compatible with the tree command. Note that not all of them have short +options as they conflict with rclone's short options. + + rclone tree remote:path [flags] + +Options + + -a, --all All files are listed (list . files too). + -C, --color Turn colorization on always. + -d, --dirs-only List directories only. + --dirsfirst List directories before files (-U disables). + --full-path Print the full path prefix for each file. + -h, --help help for tree + --human Print the size in a more human readable way. + --level int Descend only level directories deep. + -D, --modtime Print the date of last modification. + -i, --noindent Don't print indentation lines. + --noreport Turn off file/directory count at end of tree listing. + -o, --output string Output to file instead of stdout. + -p, --protections Print the protections for each file. + -Q, --quote Quote filenames with double quotes. + -s, --size Print the size in bytes of each file. + --sort string Select sort: name,version,size,mtime,ctime. + --sort-ctime Sort files by last status change time. + -t, --sort-modtime Sort files by last modification time. + -r, --sort-reverse Reverse the order of the sort. + -U, --unsorted Leave files unsorted. + --version Sort files alphanumerically by version. + + +Copying single files + +rclone normally syncs or copies directories. However, if the source +remote points to a file, rclone will just copy that file. The +destination remote must point to a directory - rclone will give the +error +Failed to create file system for "remote:file": is a file not a directory +if it isn't. + +For example, suppose you have a remote with a file in called test.jpg, +then you could copy just that file like this + + rclone copy remote:test.jpg /tmp/download + +The file test.jpg will be placed inside /tmp/download. + +This is equivalent to specifying + + rclone copy --files-from /tmp/files remote: /tmp/download + +Where /tmp/files contains the single line + + test.jpg + +It is recommended to use copy when copying individual files, not sync. +They have pretty much the same effect but copy will use a lot less +memory. + + +Syntax of remote paths + +The syntax of the paths passed to the rclone command are as follows. + +/path/to/dir + +This refers to the local file system. + +On Windows only \ may be used instead of / in local paths ONLY, non +local paths must use /. + +These paths needn't start with a leading / - if they don't then they +will be relative to the current directory. + +remote:path/to/dir + +This refers to a directory path/to/dir on remote: as defined in the +config file (configured with rclone config). + +remote:/path/to/dir + +On most backends this is refers to the same directory as +remote:path/to/dir and that format should be preferred. On a very small +number of remotes (FTP, SFTP, Dropbox for business) this will refer to a +different directory. On these, paths without a leading / will refer to +your "home" directory and paths with a leading / will refer to the root. + +:backend:path/to/dir + +This is an advanced form for creating remotes on the fly. backend should +be the name or prefix of a backend (the type in the config file) and all +the configuration for the backend should be provided on the command line +(or in environment variables). + +Eg + + rclone lsd --http-url https://pub.rclone.org :http: + +Which lists all the directories in pub.rclone.org. + + +Quoting and the shell + +When you are typing commands to your computer you are using something +called the command line shell. This interprets various characters in an +OS specific way. + +Here are some gotchas which may help users unfamiliar with the shell +rules + +Linux / OSX + +If your names have spaces or shell metacharacters (eg *, ?, $, ', " etc) +then you must quote them. Use single quotes ' by default. + + rclone copy 'Important files?' remote:backup + +If you want to send a ' you will need to use ", eg + + rclone copy "O'Reilly Reviews" remote:backup + +The rules for quoting metacharacters are complicated and if you want the +full details you'll have to consult the manual page for your shell. + +Windows + +If your names have spaces in you need to put them in ", eg + + rclone copy "E:\folder name\folder name\folder name" remote:backup + +If you are using the root directory on its own then don't quote it (see +#464 for why), eg + + rclone copy E:\ remote:backup + + +Copying files or directories with : in the names + +rclone uses : to mark a remote name. This is, however, a valid filename +component in non-Windows OSes. The remote name parser will only search +for a : up to the first / so if you need to act on a file or directory +like this then use the full path starting with a /, or use ./ as a +current directory prefix. + +So to sync a directory called sync:me to a remote called remote: use + + rclone sync ./sync:me remote:path + +or + + rclone sync /full/path/to/sync:me remote:path + + +Server Side Copy + +Most remotes (but not all - see the overview) support server side copy. + +This means if you want to copy one folder to another then rclone won't +download all the files and re-upload them; it will instruct the server +to copy them in place. + +Eg + + rclone copy s3:oldbucket s3:newbucket + +Will copy the contents of oldbucket to newbucket without downloading and +re-uploading. + +Remotes which don't support server side copy WILL download and re-upload +in this case. + +Server side copies are used with sync and copy and will be identified in +the log when using the -v flag. The move command may also use them if +remote doesn't support server side move directly. This is done by +issuing a server side copy then a delete which is much quicker than a +download and re-upload. + +Server side copies will only be attempted if the remote names are the +same. + +This can be used when scripting to make aged backups efficiently, eg + + rclone sync remote:current-backup remote:previous-backup + rclone sync /path/to/files remote:current-backup + + +Options + +Rclone has a number of options to control its behaviour. + +Options which use TIME use the go time parser. A duration string is a +possibly signed sequence of decimal numbers, each with optional fraction +and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units +are "ns", "us" (or "µs"), "ms", "s", "m", "h". + +Options which use SIZE use kByte by default. However, a suffix of b for +bytes, k for kBytes, M for MBytes, G for GBytes, T for TBytes and P for +PBytes may be used. These are the binary units, eg 1, 2**10, 2**20, +2**30 respectively. + +--backup-dir=DIR + +When using sync, copy or move any files which would have been +overwritten or deleted are moved in their original hierarchy into this +directory. + +If --suffix is set, then the moved files will have the suffix added to +them. If there is a file with the same path (after the suffix has been +added) in DIR, then it will be overwritten. + +The remote in use must support server side move or copy and you must use +the same remote as the destination of the sync. The backup directory +must not overlap the destination directory. + +For example + + rclone sync /path/to/local remote:current --backup-dir remote:old + +will sync /path/to/local to remote:current, but for any files which +would have been updated or deleted will be stored in remote:old. + +If running rclone from a script you might want to use today's date as +the directory name passed to --backup-dir to store the old files, or you +might want to pass --suffix with today's date. + +--bind string + +Local address to bind to for outgoing connections. This can be an IPv4 +address (1.2.3.4), an IPv6 address (1234::789A) or host name. If the +host name doesn't resolve or resolves to more than one IP address it +will give an error. + +--bwlimit=BANDWIDTH_SPEC + +This option controls the bandwidth limit. Limits can be specified in two +ways: As a single limit, or as a timetable. + +Single limits last for the duration of the session. To use a single +limit, specify the desired bandwidth in kBytes/s, or use a suffix +b|k|M|G. The default is 0 which means to not limit bandwidth. + +For example, to limit bandwidth usage to 10 MBytes/s use --bwlimit 10M + +It is also possible to specify a "timetable" of limits, which will cause +certain limits to be applied at certain times. To specify a timetable, +format your entries as "WEEKDAY-HH:MM,BANDWIDTH +WEEKDAY-HH:MM,BANDWIDTH..." where: WEEKDAY is optional element. It could +be writen as whole world or only using 3 first characters. HH:MM is an +hour from 00:00 to 23:59. + +An example of a typical timetable to avoid link saturation during +daytime working hours could be: + +--bwlimit "08:00,512 12:00,10M 13:00,512 18:00,30M 23:00,off" + +In this example, the transfer bandwidth will be every day set to +512kBytes/sec at 8am. At noon, it will raise to 10Mbytes/s, and drop +back to 512kBytes/sec at 1pm. At 6pm, the bandwidth limit will be set to +30MBytes/s, and at 11pm it will be completely disabled (full speed). +Anything between 11pm and 8am will remain unlimited. + +An example of timetable with WEEKDAY could be: + +--bwlimit "Mon-00:00,512 Fri-23:59,10M Sat-10:00,1M Sun-20:00,off" + +It mean that, the transfer bandwidh will be set to 512kBytes/sec on +Monday. It will raise to 10Mbytes/s before the end of Friday. At 10:00 +on Sunday it will be set to 1Mbyte/s. From 20:00 at Sunday will be +unlimited. + +Timeslots without weekday are extended to whole week. So this one +example: + +--bwlimit "Mon-00:00,512 12:00,1M Sun-20:00,off" + +Is equal to this: + +--bwlimit "Mon-00:00,512Mon-12:00,1M Tue-12:00,1M Wed-12:00,1M Thu-12:00,1M Fri-12:00,1M Sat-12:00,1M Sun-12:00,1M Sun-20:00,off" + +Bandwidth limits only apply to the data transfer. They don't apply to +the bandwidth of the directory listings etc. + +Note that the units are Bytes/s, not Bits/s. Typically connections are +measured in Bits/s - to convert divide by 8. For example, let's say you +have a 10 Mbit/s connection and you wish rclone to use half of it - 5 +Mbit/s. This is 5/8 = 0.625MByte/s so you would use a --bwlimit 0.625M +parameter for rclone. + +On Unix systems (Linux, MacOS, …) the bandwidth limiter can be toggled +by sending a SIGUSR2 signal to rclone. This allows to remove the +limitations of a long running rclone transfer and to restore it back to +the value specified with --bwlimit quickly when needed. Assuming there +is only one rclone instance running, you can toggle the limiter like +this: + + kill -SIGUSR2 $(pidof rclone) + +If you configure rclone with a remote control then you can use change +the bwlimit dynamically: + + rclone rc core/bwlimit rate=1M + +--buffer-size=SIZE + +Use this sized buffer to speed up file transfers. Each --transfer will +use this much memory for buffering. + +When using mount or cmount each open file descriptor will use this much +memory for buffering. See the mount documentation for more details. + +Set to 0 to disable the buffering for the minimum memory usage. + +--checkers=N + +The number of checkers to run in parallel. Checkers do the equality +checking of files during a sync. For some storage systems (eg S3, Swift, +Dropbox) this can take a significant amount of time so they are run in +parallel. + +The default is to run 8 checkers in parallel. + +-c, --checksum + +Normally rclone will look at modification time and size of files to see +if they are equal. If you set this flag then rclone will check the file +hash and size to determine if files are equal. + +This is useful when the remote doesn't support setting modified time and +a more accurate sync is desired than just checking the file size. + +This is very useful when transferring between remotes which store the +same hash type on the object, eg Drive and Swift. For details of which +remotes support which hash type see the table in the overview section. + +Eg rclone --checksum sync s3:/bucket swift:/bucket would run much +quicker than without the --checksum flag. + +When using this flag, rclone won't update mtimes of remote files if they +are incorrect as it would normally. + +--config=CONFIG_FILE + +Specify the location of the rclone config file. + +Normally the config file is in your home directory as a file called +.config/rclone/rclone.conf (or .rclone.conf if created with an older +version). If $XDG_CONFIG_HOME is set it will be at +$XDG_CONFIG_HOME/rclone/rclone.conf + +If you run rclone -h and look at the help for the --config option you +will see where the default location is for you. + +Use this flag to override the config location, eg +rclone --config=".myconfig" .config. + +--contimeout=TIME + +Set the connection timeout. This should be in go time format which looks +like 5s for 5 seconds, 10m for 10 minutes, or 3h30m. + +The connection timeout is the amount of time rclone will wait for a +connection to go through to a remote object storage system. It is 1m by +default. + +--dedupe-mode MODE + +Mode to run dedupe command in. One of interactive, skip, first, newest, +oldest, rename. The default is interactive. See the dedupe command for +more information as to what these options mean. + +--disable FEATURE,FEATURE,... + +This disables a comma separated list of optional features. For example +to disable server side move and server side copy use: + + --disable move,copy + +The features can be put in in any case. + +To see a list of which features can be disabled use: + + --disable help + +See the overview features and optional features to get an idea of which +feature does what. + +This flag can be useful for debugging and in exceptional circumstances +(eg Google Drive limiting the total volume of Server Side Copies to +100GB/day). + +-n, --dry-run + +Do a trial run with no permanent changes. Use this to see what rclone +would do without actually doing it. Useful when setting up the sync +command which deletes files in the destination. + +--ignore-checksum + +Normally rclone will check that the checksums of transferred files +match, and give an error "corrupted on transfer" if they don't. + +You can use this option to skip that check. You should only use it if +you have had the "corrupted on transfer" error message and you are sure +you might want to transfer potentially corrupted data. + +--ignore-existing + +Using this option will make rclone unconditionally skip all files that +exist on the destination, no matter the content of these files. + +While this isn't a generally recommended option, it can be useful in +cases where your files change due to encryption. However, it cannot +correct partial transfers in case a transfer was interrupted. + +--ignore-size + +Normally rclone will look at modification time and size of files to see +if they are equal. If you set this flag then rclone will check only the +modification time. If --checksum is set then it only checks the +checksum. + +It will also cause rclone to skip verifying the sizes are the same after +transfer. + +This can be useful for transferring files to and from OneDrive which +occasionally misreports the size of image files (see #399 for more +info). + +-I, --ignore-times + +Using this option will cause rclone to unconditionally upload all files +regardless of the state of files on the destination. + +Normally rclone would skip any files that have the same modification +time and are the same size (or have the same checksum if using +--checksum). + +--immutable + +Treat source and destination files as immutable and disallow +modification. + +With this option set, files will be created and deleted as requested, +but existing files will never be updated. If an existing file does not +match between the source and destination, rclone will give the error +Source and destination exist but do not match: immutable file modified. + +Note that only commands which transfer files (e.g. sync, copy, move) are +affected by this behavior, and only modification is disallowed. Files +may still be deleted explicitly (e.g. delete, purge) or implicitly (e.g. +sync, move). Use copy --immutable if it is desired to avoid deletion as +well as modification. + +This can be useful as an additional layer of protection for immutable or +append-only data sets (notably backup archives), where modification +implies corruption and should not be propagated. + + +--leave-root + +During rmdirs it will not remove root directory, even if it's empty. + +--log-file=FILE + +Log all of rclone's output to FILE. This is not active by default. This +can be useful for tracking down problems with syncs in combination with +the -v flag. See the Logging section for more info. + +Note that if you are using the logrotate program to manage rclone's +logs, then you should use the copytruncate option as rclone doesn't have +a signal to rotate logs. + +--log-level LEVEL + +This sets the log level for rclone. The default log level is NOTICE. + +DEBUG is equivalent to -vv. It outputs lots of debug info - useful for +bug reports and really finding out what rclone is doing. + +INFO is equivalent to -v. It outputs information about each transfer and +prints stats once a minute by default. + +NOTICE is the default log level if no logging flags are supplied. It +outputs very little when things are working normally. It outputs +warnings and significant events. + +ERROR is equivalent to -q. It only outputs error messages. + +--low-level-retries NUMBER + +This controls the number of low level retries rclone does. + +A low level retry is used to retry a failing operation - typically one +HTTP request. This might be uploading a chunk of a big file for example. +You will see low level retries in the log with the -v flag. + +This shouldn't need to be changed from the default in normal operations. +However, if you get a lot of low level retries you may wish to reduce +the value so rclone moves on to a high level retry (see the --retries +flag) quicker. + +Disable low level retries with --low-level-retries 1. + +--max-backlog=N + +This is the maximum allowable backlog of files in a sync/copy/move +queued for being checked or transferred. + +This can be set arbitrarily large. It will only use memory when the +queue is in use. Note that it will use in the order of N kB of memory +when the backlog is in use. + +Setting this large allows rclone to calculate how many files are pending +more accurately and give a more accurate estimated finish time. + +Setting this small will make rclone more synchronous to the listings of +the remote which may be desirable. + +--max-delete=N + +This tells rclone not to delete more than N files. If that limit is +exceeded then a fatal error will be generated and rclone will stop the +operation in progress. + +--max-depth=N + +This modifies the recursion depth for all the commands except purge. + +So if you do rclone --max-depth 1 ls remote:path you will see only the +files in the top level directory. Using --max-depth 2 means you will see +all the files in first two directory levels and so on. + +For historical reasons the lsd command defaults to using a --max-depth +of 1 - you can override this with the command line flag. + +You can use this command to disable recursion (with --max-depth 1). + +Note that if you use this with sync and --delete-excluded the files not +recursed through are considered excluded and will be deleted on the +destination. Test first with --dry-run if you are not sure what will +happen. + +--max-transfer=SIZE + +Rclone will stop transferring when it has reached the size specified. +Defaults to off. + +When the limit is reached all transfers will stop immediately. + +Rclone will exit with exit code 8 if the transfer limit is reached. + +--modify-window=TIME + +When checking whether a file has been modified, this is the maximum +allowed time difference that a file can have and still be considered +equivalent. + +The default is 1ns unless this is overridden by a remote. For example OS +X only stores modification times to the nearest second so if you are +reading and writing to an OS X filing system this will be 1s by default. + +This command line flag allows you to override that computed default. + +--no-gzip-encoding + +Don't set Accept-Encoding: gzip. This means that rclone won't ask the +server for compressed files automatically. Useful if you've set the +server to return files with Content-Encoding: gzip but you uploaded +compressed files. + +There is no need to set this in normal operation, and doing so will +decrease the network transfer efficiency of rclone. + +--no-update-modtime + +When using this flag, rclone won't update modification times of remote +files if they are incorrect as it would normally. + +This can be used if the remote is being synced with another tool also +(eg the Google Drive client). + +--P, --progress + +This flag makes rclone update the stats in a static block in the +terminal providing a realtime overview of the transfer. + +Any log messages will scroll above the static block. Log messages will +push the static block down to the bottom of the terminal where it will +stay. + +Normally this is updated every 500mS but this period can be overridden +with the --stats flag. + +This can be used with the --stats-one-line flag for a simpler display. + +-q, --quiet + +Normally rclone outputs stats and a completion message. If you set this +flag it will make as little output as possible. + +--retries int + +Retry the entire sync if it fails this many times it fails (default 3). + +Some remotes can be unreliable and a few retries help pick up the files +which didn't get transferred because of errors. + +Disable retries with --retries 1. + +--retries-sleep=TIME + +This sets the interval between each retry specified by --retries + +The default is 0. Use 0 to disable. + +--size-only + +Normally rclone will look at modification time and size of files to see +if they are equal. If you set this flag then rclone will check only the +size. + +This can be useful transferring files from Dropbox which have been +modified by the desktop sync client which doesn't set checksums of +modification times in the same way as rclone. + +--stats=TIME + +Commands which transfer data (sync, copy, copyto, move, moveto) will +print data transfer stats at regular intervals to show their progress. + +This sets the interval. + +The default is 1m. Use 0 to disable. + +If you set the stats interval then all commands can show stats. This can +be useful when running other commands, check or mount for example. + +Stats are logged at INFO level by default which means they won't show at +default log level NOTICE. Use --stats-log-level NOTICE or -v to make +them show. See the Logging section for more info on log levels. + +Note that on macOS you can send a SIGINFO (which is normally ctrl-T in +the terminal) to make the stats print immediately. + +--stats-file-name-length integer + +By default, the --stats output will truncate file names and paths longer +than 40 characters. This is equivalent to providing +--stats-file-name-length 40. Use --stats-file-name-length 0 to disable +any truncation of file names printed by stats. + +--stats-log-level string + +Log level to show --stats output at. This can be DEBUG, INFO, NOTICE, or +ERROR. The default is INFO. This means at the default level of logging +which is NOTICE the stats won't show - if you want them to then use +--stats-log-level NOTICE. See the Logging section for more info on log +levels. + +--stats-one-line + +When this is specified, rclone condenses the stats into a single line +showing the most important stats only. + +--stats-unit=bits|bytes + +By default, data transfer rates will be printed in bytes/second. + +This option allows the data rate to be printed in bits/second. + +Data transfer volume will still be reported in bytes. + +The rate is reported as a binary unit, not SI unit. So 1 Mbit/s equals +1,048,576 bits/s and not 1,000,000 bits/s. + +The default is bytes. + +--suffix=SUFFIX + +This is for use with --backup-dir only. If this isn't set then +--backup-dir will move files with their original name. If it is set then +the files will have SUFFIX added on to them. + +See --backup-dir for more info. + +--syslog + +On capable OSes (not Windows or Plan9) send all log output to syslog. + +This can be useful for running rclone in a script or rclone mount. + +--syslog-facility string + +If using --syslog this sets the syslog facility (eg KERN, USER). See +man syslog for a list of possible facilities. The default facility is +DAEMON. + +--tpslimit float + +Limit HTTP transactions per second to this. Default is 0 which is used +to mean unlimited transactions per second. + +For example to limit rclone to 10 HTTP transactions per second use +--tpslimit 10, or to 1 transaction every 2 seconds use --tpslimit 0.5. + +Use this when the number of transactions per second from rclone is +causing a problem with the cloud storage provider (eg getting you banned +or rate limited). + +This can be very useful for rclone mount to control the behaviour of +applications using it. + +See also --tpslimit-burst. + +--tpslimit-burst int + +Max burst of transactions for --tpslimit. (default 1) + +Normally --tpslimit will do exactly the number of transaction per second +specified. However if you supply --tps-burst then rclone can save up +some transactions from when it was idle giving a burst of up to the +parameter supplied. + +For example if you provide --tpslimit-burst 10 then if rclone has been +idle for more than 10*--tpslimit then it can do 10 transactions very +quickly before they are limited again. + +This may be used to increase performance of --tpslimit without changing +the long term average number of transactions per second. + +--track-renames + +By default, rclone doesn't keep track of renamed files, so if you rename +a file locally then sync it to a remote, rclone will delete the old file +on the remote and upload a new copy. + +If you use this flag, and the remote supports server side copy or server +side move, and the source and destination have a compatible hash, then +this will track renames during sync operations and perform renaming +server-side. + +Files will be matched by size and hash - if both match then a rename +will be considered. + +If the destination does not support server-side copy or move, rclone +will fall back to the default behaviour and log an error level message +to the console. + +Note that --track-renames uses extra memory to keep track of all the +rename candidates. + +Note also that --track-renames is incompatible with --delete-before and +will select --delete-after instead of --delete-during. + +--delete-(before,during,after) + +This option allows you to specify when files on your destination are +deleted when you sync folders. + +Specifying the value --delete-before will delete all files present on +the destination, but not on the source _before_ starting the transfer of +any new or updated files. This uses two passes through the file systems, +one for the deletions and one for the copies. + +Specifying --delete-during will delete files while checking and +uploading files. This is the fastest option and uses the least memory. + +Specifying --delete-after (the default value) will delay deletion of +files until all new/updated files have been successfully transferred. +The files to be deleted are collected in the copy pass then deleted +after the copy pass has completed successfully. The files to be deleted +are held in memory so this mode may use more memory. This is the safest +mode as it will only delete files if there have been no errors +subsequent to that. If there have been errors before the deletions start +then you will get the message +not deleting files as there were IO errors. + +--fast-list + +When doing anything which involves a directory listing (eg sync, copy, +ls - in fact nearly every command), rclone normally lists a directory +and processes it before using more directory lists to process any +subdirectories. This can be parallelised and works very quickly using +the least amount of memory. + +However, some remotes have a way of listing all files beneath a +directory in one (or a small number) of transactions. These tend to be +the bucket based remotes (eg S3, B2, GCS, Swift, Hubic). + +If you use the --fast-list flag then rclone will use this method for +listing directories. This will have the following consequences for the +listing: + +- It WILL use fewer transactions (important if you pay for them) +- It WILL use more memory. Rclone has to load the whole listing into + memory. +- It _may_ be faster because it uses fewer transactions +- It _may_ be slower because it can't be parallelized + +rclone should always give identical results with and without +--fast-list. + +If you pay for transactions and can fit your entire sync listing into +memory then --fast-list is recommended. If you have a very big sync to +do then don't use --fast-list otherwise you will run out of memory. + +If you use --fast-list on a remote which doesn't support it, then rclone +will just ignore it. + +--timeout=TIME + +This sets the IO idle timeout. If a transfer has started but then +becomes idle for this long it is considered broken and disconnected. + +The default is 5m. Set to 0 to disable. + +--transfers=N + +The number of file transfers to run in parallel. It can sometimes be +useful to set this to a smaller number if the remote is giving a lot of +timeouts or bigger if you have lots of bandwidth and a fast remote. + +The default is to run 4 file transfers in parallel. + +-u, --update + +This forces rclone to skip any files which exist on the destination and +have a modified time that is newer than the source file. + +If an existing destination file has a modification time equal (within +the computed modify window precision) to the source file's, it will be +updated if the sizes are different. + +On remotes which don't support mod time directly the time checked will +be the uploaded time. This means that if uploading to one of these +remotes, rclone will skip any files which exist on the destination and +have an uploaded time that is newer than the modification time of the +source file. + +This can be useful when transferring to a remote which doesn't support +mod times directly as it is more accurate than a --size-only check and +faster than using --checksum. + +--use-server-modtime + +Some object-store backends (e.g, Swift, S3) do not preserve file +modification times (modtime). On these backends, rclone stores the +original modtime as additional metadata on the object. By default it +will make an API call to retrieve the metadata when the modtime is +needed by an operation. + +Use this flag to disable the extra API call and rely instead on the +server's modified time. In cases such as a local to remote sync, knowing +the local file is newer than the time it was last uploaded to the remote +is sufficient. In those cases, this flag can speed up the process and +reduce the number of API calls necessary. + +-v, -vv, --verbose + +With -v rclone will tell you about each file that is transferred and a +small number of significant events. + +With -vv rclone will become very verbose telling you about every file it +considers and transfers. Please send bug reports with a log with this +setting. + +-V, --version + +Prints the version number + + +Configuration Encryption + +Your configuration file contains information for logging in to your +cloud services. This means that you should keep your .rclone.conf file +in a secure location. + +If you are in an environment where that isn't possible, you can add a +password to your configuration. This means that you will have to enter +the password every time you start rclone. + +To add a password to your rclone configuration, execute rclone config. + + >rclone config + Current remotes: + + e) Edit existing remote + n) New remote + d) Delete remote + s) Set configuration password + q) Quit config + e/n/d/s/q> + +Go into s, Set configuration password: + + e/n/d/s/q> s + Your configuration is not encrypted. + If you add a password, you will protect your login information to cloud services. + a) Add Password + q) Quit to main menu + a/q> a + Enter NEW configuration password: + password: + Confirm NEW password: + password: + Password set + Your configuration is encrypted. + c) Change Password + u) Unencrypt configuration + q) Quit to main menu + c/u/q> + +Your configuration is now encrypted, and every time you start rclone you +will now be asked for the password. In the same menu, you can change the +password or completely remove encryption from your configuration. + +There is no way to recover the configuration if you lose your password. + +rclone uses nacl secretbox which in turn uses XSalsa20 and Poly1305 to +encrypt and authenticate your configuration with secret-key +cryptography. The password is SHA-256 hashed, which produces the key for +secretbox. The hashed password is not stored. + +While this provides very good security, we do not recommend storing your +encrypted rclone configuration in public if it contains sensitive +information, maybe except if you use a very strong password. + +If it is safe in your environment, you can set the RCLONE_CONFIG_PASS +environment variable to contain your password, in which case it will be +used for decrypting the configuration. + +You can set this for a session from a script. For unix like systems save +this to a file called set-rclone-password: + + #!/bin/echo Source this file don't run it + + read -s RCLONE_CONFIG_PASS + export RCLONE_CONFIG_PASS + +Then source the file when you want to use it. From the shell you would +do source set-rclone-password. It will then ask you for the password and +set it in the environment variable. + +If you are running rclone inside a script, you might want to disable +password prompts. To do that, pass the parameter --ask-password=false to +rclone. This will make rclone fail instead of asking for a password if +RCLONE_CONFIG_PASS doesn't contain a valid password. + + +Developer options + +These options are useful when developing or debugging rclone. There are +also some more remote specific options which aren't documented here +which are used for testing. These start with remote name eg +--drive-test-option - see the docs for the remote in question. + +--cpuprofile=FILE + +Write CPU profile to file. This can be analysed with go tool pprof. + +--dump flag,flag,flag + +The --dump flag takes a comma separated list of flags to dump info +about. These are: + +--dump headers + +Dump HTTP headers with Authorization: lines removed. May still contain +sensitive info. Can be very verbose. Useful for debugging only. + +Use --dump auth if you do want the Authorization: headers. + +--dump bodies + +Dump HTTP headers and bodies - may contain sensitive info. Can be very +verbose. Useful for debugging only. + +Note that the bodies are buffered in memory so don't use this for +enormous files. + +--dump requests + +Like --dump bodies but dumps the request bodies and the response +headers. Useful for debugging download problems. + +--dump responses + +Like --dump bodies but dumps the response bodies and the request +headers. Useful for debugging upload problems. + +--dump auth + +Dump HTTP headers - will contain sensitive info such as Authorization: +headers - use --dump headers to dump without Authorization: headers. Can +be very verbose. Useful for debugging only. + +--dump filters + +Dump the filters to the output. Useful to see exactly what include and +exclude options are filtering on. + +--dump goroutines + +This dumps a list of the running go-routines at the end of the command +to standard output. + +--dump openfiles + +This dumps a list of the open files at the end of the command. It uses +the lsof command to do that so you'll need that installed to use it. + +--memprofile=FILE + +Write memory profile to file. This can be analysed with go tool pprof. + +--no-check-certificate=true/false + +--no-check-certificate controls whether a client verifies the server's +certificate chain and host name. If --no-check-certificate is true, TLS +accepts any certificate presented by the server and any host name in +that certificate. In this mode, TLS is susceptible to man-in-the-middle +attacks. + +This option defaults to false. + +THIS SHOULD BE USED ONLY FOR TESTING. + + +Filtering + +For the filtering options + +- --delete-excluded +- --filter +- --filter-from +- --exclude +- --exclude-from +- --include +- --include-from +- --files-from +- --min-size +- --max-size +- --min-age +- --max-age +- --dump filters + +See the filtering section. + + +Remote control + +For the remote control options and for instructions on how to remote +control rclone + +- --rc +- and anything starting with --rc- + +See the remote control section. + + +Logging + +rclone has 4 levels of logging, ERROR, NOTICE, INFO and DEBUG. + +By default, rclone logs to standard error. This means you can redirect +standard error and still see the normal output of rclone commands (eg +rclone ls). + +By default, rclone will produce Error and Notice level messages. + +If you use the -q flag, rclone will only produce Error messages. + +If you use the -v flag, rclone will produce Error, Notice and Info +messages. + +If you use the -vv flag, rclone will produce Error, Notice, Info and +Debug messages. + +You can also control the log levels with the --log-level flag. + +If you use the --log-file=FILE option, rclone will redirect Error, Info +and Debug messages along with standard error to FILE. + +If you use the --syslog flag then rclone will log to syslog and the +--syslog-facility control which facility it uses. + +Rclone prefixes all log messages with their level in capitals, eg INFO +which makes it easy to grep the log file for different kinds of +information. + + +Exit Code + +If any errors occur during the command execution, rclone will exit with +a non-zero exit code. This allows scripts to detect when rclone +operations have failed. + +During the startup phase, rclone will exit immediately if an error is +detected in the configuration. There will always be a log message +immediately before exiting. + +When rclone is running it will accumulate errors as it goes along, and +only exit with a non-zero exit code if (after retries) there were still +failed transfers. For every error counted there will be a high priority +log message (visible with -q) showing the message and which file caused +the problem. A high priority message is also shown when starting a retry +so the user can see that any previous error messages may not be valid +after the retry. If rclone has done a retry it will log a high priority +message if the retry was successful. + +List of exit codes + +- 0 - success +- 1 - Syntax or usage error +- 2 - Error not otherwise categorised +- 3 - Directory not found +- 4 - File not found +- 5 - Temporary error (one that more retries might fix) (Retry errors) +- 6 - Less serious errors (like 461 errors from dropbox) (NoRetry + errors) +- 7 - Fatal error (one that more retries won't fix, like account + suspended) (Fatal errors) +- 8 - Transfer exceeded - limit set by --max-transfer reached + + +Environment Variables + +Rclone can be configured entirely using environment variables. These can +be used to set defaults for options or config file entries. + +Options + +Every option in rclone can have its default set by environment variable. + +To find the name of the environment variable, first, take the long +option name, strip the leading --, change - to _, make upper case and +prepend RCLONE_. + +For example, to always set --stats 5s, set the environment variable +RCLONE_STATS=5s. If you set stats on the command line this will override +the environment variable setting. + +Or to always use the trash in drive --drive-use-trash, set +RCLONE_DRIVE_USE_TRASH=true. + +The same parser is used for the options and the environment variables so +they take exactly the same form. + +Config file + +You can set defaults for values in the config file on an individual +remote basis. If you want to use this feature, you will need to discover +the name of the config items that you want. The easiest way is to run +through rclone config by hand, then look in the config file to see what +the values are (the config file can be found by looking at the help for +--config in rclone help). + +To find the name of the environment variable, you need to set, take +RCLONE_CONFIG_ + name of remote + _ + name of config file option and +make it all uppercase. + +For example, to configure an S3 remote named mys3: without a config file +(using unix ways of setting environment variables): + + $ export RCLONE_CONFIG_MYS3_TYPE=s3 + $ export RCLONE_CONFIG_MYS3_ACCESS_KEY_ID=XXX + $ export RCLONE_CONFIG_MYS3_SECRET_ACCESS_KEY=XXX + $ rclone lsd MYS3: + -1 2016-09-21 12:54:21 -1 my-bucket + $ rclone listremotes | grep mys3 + mys3: + +Note that if you want to create a remote using environment variables you +must create the ..._TYPE variable as above. + +Other environment variables + +- RCLONE_CONFIG_PASS` set to contain your config file password (see + Configuration Encryption section) +- HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions + thereof). + - HTTPS_PROXY takes precedence over HTTP_PROXY for https requests. + - The environment values may be either a complete URL or a + "host[:port]" for, in which case the "http" scheme is assumed. + + + +CONFIGURING RCLONE ON A REMOTE / HEADLESS MACHINE + + +Some of the configurations (those involving oauth2) require an Internet +connected web browser. + +If you are trying to set rclone up on a remote or headless box with no +browser available on it (eg a NAS or a server in a datacenter) then you +will need to use an alternative means of configuration. There are two +ways of doing it, described below. + + +Configuring using rclone authorize + +On the headless box + + ... + Remote config + Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine + y) Yes + n) No + y/n> n + For this to work, you will need rclone available on a machine that has a web browser available. + Execute the following on your machine: + rclone authorize "amazon cloud drive" + Then paste the result below: + result> + +Then on your main desktop machine + + rclone authorize "amazon cloud drive" + If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth + Log in and authorize rclone for access + Waiting for code... + Got code + Paste the following into your remote machine ---> + SECRET_TOKEN + <---End paste + +Then back to the headless box, paste in the code + + result> SECRET_TOKEN + -------------------- + [acd12] + client_id = + client_secret = + token = SECRET_TOKEN + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> + + +Configuring by copying the config file + +Rclone stores all of its config in a single configuration file. This can +easily be copied to configure a remote rclone. + +So first configure rclone on your desktop machine + + rclone config + +to set up the config file. + +Find the config file by running rclone -h and looking for the help for +the --config option + + $ rclone -h + [snip] + --config="/home/user/.rclone.conf": Config file. + [snip] + +Now transfer it to the remote box (scp, cut paste, ftp, sftp etc) and +place it in the correct place (use rclone -h on the remote box to find +out where). + + + +FILTERING, INCLUDES AND EXCLUDES + + +Rclone has a sophisticated set of include and exclude rules. Some of +these are based on patterns and some on other things like file size. + +The filters are applied for the copy, sync, move, ls, lsl, md5sum, +sha1sum, size, delete and check operations. Note that purge does not +obey the filters. + +Each path as it passes through rclone is matched against the include and +exclude rules like --include, --exclude, --include-from, --exclude-from, +--filter, or --filter-from. The simplest way to try them out is using +the ls command, or --dry-run together with -v. + + +Patterns + +The patterns used to match files for inclusion or exclusion are based on +"file globs" as used by the unix shell. + +If the pattern starts with a / then it only matches at the top level of +the directory tree, RELATIVE TO THE ROOT OF THE REMOTE (not necessarily +the root of the local drive). If it doesn't start with / then it is +matched starting at the END OF THE PATH, but it will only match a +complete path element: + + file.jpg - matches "file.jpg" + - matches "directory/file.jpg" + - doesn't match "afile.jpg" + - doesn't match "directory/afile.jpg" + /file.jpg - matches "file.jpg" in the root directory of the remote + - doesn't match "afile.jpg" + - doesn't match "directory/file.jpg" + +IMPORTANT Note that you must use / in patterns and not \ even if running +on Windows. + +A * matches anything but not a /. + + *.jpg - matches "file.jpg" + - matches "directory/file.jpg" + - doesn't match "file.jpg/something" + +Use ** to match anything, including slashes (/). + + dir/** - matches "dir/file.jpg" + - matches "dir/dir1/dir2/file.jpg" + - doesn't match "directory/file.jpg" + - doesn't match "adir/file.jpg" + +A ? matches any character except a slash /. + + l?ss - matches "less" + - matches "lass" + - doesn't match "floss" + +A [ and ] together make a a character class, such as [a-z] or [aeiou] or +[[:alpha:]]. See the go regexp docs for more info on these. + + h[ae]llo - matches "hello" + - matches "hallo" + - doesn't match "hullo" + +A { and } define a choice between elements. It should contain a comma +separated list of patterns, any of which might match. These patterns can +contain wildcards. + + {one,two}_potato - matches "one_potato" + - matches "two_potato" + - doesn't match "three_potato" + - doesn't match "_potato" + +Special characters can be escaped with a \ before them. + + \*.jpg - matches "*.jpg" + \\.jpg - matches "\.jpg" + \[one\].jpg - matches "[one].jpg" + +Note also that rclone filter globs can only be used in one of the filter +command line flags, not in the specification of the remote, so +rclone copy "remote:dir*.jpg" /path/to/dir won't work - what is required +is rclone --include "*.jpg" copy remote:dir /path/to/dir + +Directories + +Rclone keeps track of directories that could match any file patterns. + +Eg if you add the include rule + + /a/*.jpg + +Rclone will synthesize the directory include rule + + /a/ + +If you put any rules which end in / then it will only match directories. + +Directory matches are ONLY used to optimise directory access patterns - +you must still match the files that you want to match. Directory matches +won't optimise anything on bucket based remotes (eg s3, swift, google +compute storage, b2) which don't have a concept of directory. + +Differences between rsync and rclone patterns + +Rclone implements bash style {a,b,c} glob matching which rsync doesn't. + +Rclone always does a wildcard match so \ must always escape a \. + + +How the rules are used + +Rclone maintains a combined list of include rules and exclude rules. + +Each file is matched in order, starting from the top, against the rule +in the list until it finds a match. The file is then included or +excluded according to the rule type. + +If the matcher fails to find a match after testing against all the +entries in the list then the path is included. + +For example given the following rules, + being include, - being exclude, + + - secret*.jpg + + *.jpg + + *.png + + file2.avi + - * + +This would include + +- file1.jpg +- file3.png +- file2.avi + +This would exclude + +- secret17.jpg +- non *.jpg and *.png + +A similar process is done on directory entries before recursing into +them. This only works on remotes which have a concept of directory (Eg +local, google drive, onedrive, amazon drive) and not on bucket based +remotes (eg s3, swift, google compute storage, b2). + + +Adding filtering rules + +Filtering rules are added with the following command line flags. + +Repeating options + +You can repeat the following options to add more than one rule of that +type. + +- --include +- --include-from +- --exclude +- --exclude-from +- --filter +- --filter-from + +IMPORTANT You should not use --include* together with --exclude*. It may +produce different results than you expected. In that case try to use: +--filter*. + +Note that all the options of the same type are processed together in the +order above, regardless of what order they were placed on the command +line. + +So all --include options are processed first in the order they appeared +on the command line, then all --include-from options etc. + +To mix up the order includes and excludes, the --filter flag can be +used. + +--exclude - Exclude files matching pattern + +Add a single exclude rule with --exclude. + +This flag can be repeated. See above for the order the flags are +processed in. + +Eg --exclude *.bak to exclude all bak files from the sync. + +--exclude-from - Read exclude patterns from file + +Add exclude rules from a file. + +This flag can be repeated. See above for the order the flags are +processed in. + +Prepare a file like this exclude-file.txt + + # a sample exclude rule file + *.bak + file2.jpg + +Then use as --exclude-from exclude-file.txt. This will sync all files +except those ending in bak and file2.jpg. + +This is useful if you have a lot of rules. + +--include - Include files matching pattern + +Add a single include rule with --include. + +This flag can be repeated. See above for the order the flags are +processed in. + +Eg --include *.{png,jpg} to include all png and jpg files in the backup +and no others. + +This adds an implicit --exclude * at the very end of the filter list. +This means you can mix --include and --include-from with the other +filters (eg --exclude) but you must include all the files you want in +the include statement. If this doesn't provide enough flexibility then +you must use --filter-from. + +--include-from - Read include patterns from file + +Add include rules from a file. + +This flag can be repeated. See above for the order the flags are +processed in. + +Prepare a file like this include-file.txt + + # a sample include rule file + *.jpg + *.png + file2.avi + +Then use as --include-from include-file.txt. This will sync all jpg, png +files and file2.avi. + +This is useful if you have a lot of rules. + +This adds an implicit --exclude * at the very end of the filter list. +This means you can mix --include and --include-from with the other +filters (eg --exclude) but you must include all the files you want in +the include statement. If this doesn't provide enough flexibility then +you must use --filter-from. + +--filter - Add a file-filtering rule + +This can be used to add a single include or exclude rule. Include rules +start with + and exclude rules start with -. A special rule called ! can +be used to clear the existing rules. + +This flag can be repeated. See above for the order the flags are +processed in. + +Eg --filter "- *.bak" to exclude all bak files from the sync. + +--filter-from - Read filtering patterns from a file + +Add include/exclude rules from a file. + +This flag can be repeated. See above for the order the flags are +processed in. + +Prepare a file like this filter-file.txt + + # a sample filter rule file + - secret*.jpg + + *.jpg + + *.png + + file2.avi + - /dir/Trash/** + + /dir/** + # exclude everything else + - * + +Then use as --filter-from filter-file.txt. The rules are processed in +the order that they are defined. + +This example will include all jpg and png files, exclude any files +matching secret*.jpg and include file2.avi. It will also include +everything in the directory dir at the root of the sync, except +dir/Trash which it will exclude. Everything else will be excluded from +the sync. + +--files-from - Read list of source-file names + +This reads a list of file names from the file passed in and ONLY these +files are transferred. The FILTERING RULES ARE IGNORED completely if you +use this option. + +This option can be repeated to read from more than one file. These are +read in the order that they are placed on the command line. + +Paths within the --files-from file will be interpreted as starting with +the root specified in the command. Leading / characters are ignored. + +For example, suppose you had files-from.txt with this content: + + # comment + file1.jpg + subdir/file2.jpg + +You could then use it like this: + + rclone copy --files-from files-from.txt /home/me/pics remote:pics + +This will transfer these files only (if they exist) + + /home/me/pics/file1.jpg → remote:pics/file1.jpg + /home/me/pics/subdir/file2.jpg → remote:pics/subdirfile1.jpg + +To take a more complicated example, let's say you had a few files you +want to back up regularly with these absolute paths: + + /home/user1/important + /home/user1/dir/file + /home/user2/stuff + +To copy these you'd find a common subdirectory - in this case /home and +put the remaining files in files-from.txt with or without leading /, eg + + user1/important + user1/dir/file + user2/stuff + +You could then copy these to a remote like this + + rclone copy --files-from files-from.txt /home remote:backup + +The 3 files will arrive in remote:backup with the paths as in the +files-from.txt like this: + + /home/user1/important → remote:backup/user1/important + /home/user1/dir/file → remote:backup/user1/dir/file + /home/user2/stuff → remote:backup/stuff + +You could of course choose / as the root too in which case your +files-from.txt might look like this. + + /home/user1/important + /home/user1/dir/file + /home/user2/stuff + +And you would transfer it like this + + rclone copy --files-from files-from.txt / remote:backup + +In this case there will be an extra home directory on the remote: + + /home/user1/important → remote:home/backup/user1/important + /home/user1/dir/file → remote:home/backup/user1/dir/file + /home/user2/stuff → remote:home/backup/stuff + +--min-size - Don't transfer any file smaller than this + +This option controls the minimum size file which will be transferred. +This defaults to kBytes but a suffix of k, M, or G can be used. + +For example --min-size 50k means no files smaller than 50kByte will be +transferred. + +--max-size - Don't transfer any file larger than this + +This option controls the maximum size file which will be transferred. +This defaults to kBytes but a suffix of k, M, or G can be used. + +For example --max-size 1G means no files larger than 1GByte will be +transferred. + +--max-age - Don't transfer any file older than this + +This option controls the maximum age of files to transfer. Give in +seconds or with a suffix of: + +- ms - Milliseconds +- s - Seconds +- m - Minutes +- h - Hours +- d - Days +- w - Weeks +- M - Months +- y - Years + +For example --max-age 2d means no files older than 2 days will be +transferred. + +--min-age - Don't transfer any file younger than this + +This option controls the minimum age of files to transfer. Give in +seconds or with a suffix (see --max-age for list of suffixes) + +For example --min-age 2d means no files younger than 2 days will be +transferred. + +--delete-excluded - Delete files on dest excluded from sync + +IMPORTANT this flag is dangerous - use with --dry-run and -v first. + +When doing rclone sync this will delete any files which are excluded +from the sync on the destination. + +If for example you did a sync from A to B without the --min-size 50k +flag + + rclone sync A: B: + +Then you repeated it like this with the --delete-excluded + + rclone --min-size 50k --delete-excluded sync A: B: + +This would delete all files on B which are less than 50 kBytes as these +are now excluded from the sync. + +Always test first with --dry-run and -v before using this flag. + +--dump filters - dump the filters to the output + +This dumps the defined filters to the output as regular expressions. + +Useful for debugging. + + +Quoting shell metacharacters + +The examples above may not work verbatim in your shell as they have +shell metacharacters in them (eg *), and may require quoting. + +Eg linux, OSX + +- --include \*.jpg +- --include '*.jpg' +- --include='*.jpg' + +In Windows the expansion is done by the command not the shell so this +should work fine + +- --include *.jpg + + +Exclude directory based on a file + +It is possible to exclude a directory based on a file, which is present +in this directory. Filename should be specified using the +--exclude-if-present flag. This flag has a priority over the other +filtering flags. + +Imagine, you have the following directory structure: + + dir1/file1 + dir1/dir2/file2 + dir1/dir2/dir3/file3 + dir1/dir2/dir3/.ignore + +You can exclude dir3 from sync by running the following command: + + rclone sync --exclude-if-present .ignore dir1 remote:backup + +Currently only one filename is supported, i.e. --exclude-if-present +should not be used multiple times. + + + +REMOTE CONTROLLING RCLONE + + +If rclone is run with the --rc flag then it starts an http server which +can be used to remote control rclone. + +NB this is experimental and everything here is subject to change! + + +Supported parameters + +--rc + +Flag to start the http server listen on remote requests + +--rc-addr=IP + +IPaddress:Port or :Port to bind server to. (default "localhost:5572") + +--rc-cert=KEY + +SSL PEM key (concatenation of certificate and CA certificate) + +--rc-client-ca=PATH + +Client certificate authority to verify clients with + +--rc-htpasswd=PATH + +htpasswd file - if not provided no authentication is done + +--rc-key=PATH + +SSL PEM Private key + +--rc-max-header-bytes=VALUE + +Maximum size of request header (default 4096) + +--rc-user=VALUE + +User name for authentication. + +--rc-pass=VALUE + +Password for authentication. + +--rc-realm=VALUE + +Realm for authentication (default "rclone") + +--rc-server-read-timeout=DURATION + +Timeout for server reading data (default 1h0m0s) + +--rc-server-write-timeout=DURATION + +Timeout for server writing data (default 1h0m0s) + + +Accessing the remote control via the rclone rc command + +Rclone itself implements the remote control protocol in its rclone rc +command. + +You can use it like this + + $ rclone rc rc/noop param1=one param2=two + { + "param1": "one", + "param2": "two" + } + +Run rclone rc on its own to see the help for the installed remote +control commands. + + +Supported commands + +cache/expire: Purge a remote from cache + +Purge a remote from the cache backend. Supports either a directory or a +file. Params: - remote = path to remote (required) - withData = +true/false to delete cached data (chunks) as well (optional) + +Eg + + rclone rc cache/expire remote=path/to/sub/folder/ + rclone rc cache/expire remote=/ withData=true + +cache/stats: Get cache stats + +Show statistics for the cache remote. + +core/bwlimit: Set the bandwidth limit. + +This sets the bandwidth limit to that passed in. + +Eg + + rclone rc core/bwlimit rate=1M + rclone rc core/bwlimit rate=off + +The format of the parameter is exactly the same as passed to --bwlimit +except only one bandwidth may be specified. + +core/gc: Runs a garbage collection. + +This tells the go runtime to do a garbage collection run. It isn't +necessary to call this normally, but it can be useful for debugging +memory problems. + +core/memstats: Returns the memory statistics + +This returns the memory statistics of the running program. What the +values mean are explained in the go docs: +https://golang.org/pkg/runtime/#MemStats + +The most interesting values for most people are: + +- HeapAlloc: This is the amount of memory rclone is actually using +- HeapSys: This is the amount of memory rclone has obtained from the + OS +- Sys: this is the total amount of memory requested from the OS +- It is virtual memory so may include unused memory + +core/pid: Return PID of current process + +This returns PID of current process. Useful for stopping rclone process. + +core/stats: Returns stats about current transfers. + +This returns all available stats + + rclone rc core/stats + +Returns the following values: + + { + "speed": average speed in bytes/sec since start of the process, + "bytes": total transferred bytes since the start of the process, + "errors": number of errors, + "checks": number of checked files, + "transfers": number of transferred files, + "deletes" : number of deleted files, + "elapsedTime": time in seconds since the start of the process, + "lastError": last occurred error, + "transferring": an array of currently active file transfers: + [ + { + "bytes": total transferred bytes for this file, + "eta": estimated time in seconds until file transfer completion + "name": name of the file, + "percentage": progress of the file transfer in percent, + "speed": speed in bytes/sec, + "speedAvg": speed in bytes/sec as an exponentially weighted moving average, + "size": size of the file in bytes + } + ], + "checking": an array of names of currently active file checks + [] + } + +Values for "transferring", "checking" and "lastError" are only assigned +if data is available. The value for "eta" is null if an eta cannot be +determined. + +rc/error: This returns an error + +This returns an error with the input as part of its error string. Useful +for testing error handling. + +rc/list: List all the registered remote control commands + +This lists all the registered remote control commands as a JSON map in +the commands response. + +rc/noop: Echo the input to the output parameters + +This echoes the input parameters to the output parameters for testing +purposes. It can be used to check that rclone is still alive and to +check that parameter passing is working properly. + +vfs/forget: Forget files or directories in the directory cache. + +This forgets the paths in the directory cache causing them to be re-read +from the remote when needed. + +If no paths are passed in then it will forget all the paths in the +directory cache. + + rclone rc vfs/forget + +Otherwise pass files or dirs in as file=path or dir=path. Any parameter +key starting with file will forget that file and any starting with dir +will forget that dir, eg + + rclone rc vfs/forget file=hello file2=goodbye dir=home/junk + +vfs/refresh: Refresh the directory cache. + +This reads the directories for the specified paths and freshens the +directory cache. + +If no paths are passed in then it will refresh the root directory. + + rclone rc vfs/refresh + +Otherwise pass directories in as dir=path. Any parameter key starting +with dir will refresh that directory, eg + + rclone rc vfs/refresh dir=home/junk dir2=data/misc + +If the parameter recursive=true is given the whole directory tree will +get refreshed. This refresh will use --fast-list if enabled. + + +Accessing the remote control via HTTP + +Rclone implements a simple HTTP based protocol. + +Each endpoint takes an JSON object and returns a JSON object or an +error. The JSON objects are essentially a map of string names to values. + +All calls must made using POST. + +The input objects can be supplied using URL parameters, POST parameters +or by supplying "Content-Type: application/json" and a JSON blob in the +body. There are examples of these below using curl. + +The response will be a JSON blob in the body of the response. This is +formatted to be reasonably human readable. + +If an error occurs then there will be an HTTP error status (usually 400) +and the body of the response will contain a JSON encoded error object. + +Using POST with URL parameters only + + curl -X POST 'http://localhost:5572/rc/noop/?potato=1&sausage=2' + +Response + + { + "potato": "1", + "sausage": "2" + } + +Here is what an error response looks like: + + curl -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2' + + { + "error": "arbitrary error on input map[potato:1 sausage:2]", + "input": { + "potato": "1", + "sausage": "2" + } + } + +Note that curl doesn't return errors to the shell unless you use the -f +option + + $ curl -f -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2' + curl: (22) The requested URL returned error: 400 Bad Request + $ echo $? + 22 + +Using POST with a form + + curl --data "potato=1" --data "sausage=2" http://localhost:5572/rc/noop/ + +Response + + { + "potato": "1", + "sausage": "2" + } + +Note that you can combine these with URL parameters too with the POST +parameters taking precedence. + + curl --data "potato=1" --data "sausage=2" "http://localhost:5572/rc/noop/?rutabaga=3&sausage=4" + +Response + + { + "potato": "1", + "rutabaga": "3", + "sausage": "4" + } + +Using POST with a JSON blob + + curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' http://localhost:5572/rc/noop/ + +response + + { + "password": "xyz", + "username": "xyz" + } + +This can be combined with URL parameters too if required. The JSON blob +takes precedence. + + curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' 'http://localhost:5572/rc/noop/?rutabaga=3&potato=4' + + { + "potato": 2, + "rutabaga": "3", + "sausage": 1 + } + + +Debugging rclone with pprof + +If you use the --rc flag this will also enable the use of the go +profiling tools on the same port. + +To use these, first install go. + +Then (for example) to profile rclone's memory use you can run: + + go tool pprof -web http://localhost:5572/debug/pprof/heap + +This should open a page in your browser showing what is using what +memory. + +You can also use the -text flag to produce a textual summary + + $ go tool pprof -text http://localhost:5572/debug/pprof/heap + Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total + flat flat% sum% cum cum% + 1024.03kB 66.62% 66.62% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.addDecoderNode + 513kB 33.38% 100% 513kB 33.38% net/http.newBufioWriterSize + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/cmd/all.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/cmd/serve.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/cmd/serve/restic.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init.0 + 0 0% 100% 1024.03kB 66.62% main.init + 0 0% 100% 513kB 33.38% net/http.(*conn).readRequest + 0 0% 100% 513kB 33.38% net/http.(*conn).serve + 0 0% 100% 1024.03kB 66.62% runtime.main + +Possible profiles to look at: + +- Memory: go tool pprof http://localhost:5572/debug/pprof/heap +- 30-second CPU profile: + go tool pprof http://localhost:5572/debug/pprof/profile +- 5-second execution trace: + wget http://localhost:5572/debug/pprof/trace?seconds=5 + +See the net/http/pprof docs for more info on how to use the profiling +and for a general overview see the Go team's blog post on profiling go +programs. + +The profiling hook is zero overhead unless it is used. + + + +OVERVIEW OF CLOUD STORAGE SYSTEMS + + +Each cloud storage system is slightly different. Rclone attempts to +provide a unified interface to them, but some underlying differences +show through. + + +Features + +Here is an overview of the major features of each cloud storage system. + + Name Hash ModTime Case Insensitive Duplicate Files MIME Type + ------------------------------ ------------- --------- ------------------ ----------------- ----------- + Amazon Drive MD5 No Yes No R + Amazon S3 MD5 Yes No No R/W + Backblaze B2 SHA1 Yes No No R/W + Box SHA1 Yes Yes No - + Dropbox DBHASH † Yes Yes No - + FTP - No No No - + Google Cloud Storage MD5 Yes No No R/W + Google Drive MD5 Yes No Yes R/W + HTTP - No No No R + Hubic MD5 Yes No No R/W + Jottacloud MD5 Yes Yes No R/W + Mega - No No Yes - + Microsoft Azure Blob Storage MD5 Yes No No R/W + Microsoft OneDrive SHA1 ‡‡ Yes Yes No R + OpenDrive MD5 Yes Yes No - + Openstack Swift MD5 Yes No No R/W + pCloud MD5, SHA1 Yes No No W + QingStor MD5 No No No R/W + SFTP MD5, SHA1 ‡ Yes Depends No - + WebDAV - Yes †† Depends No - + Yandex Disk MD5 Yes No No R/W + The local filesystem All Yes Depends No - + +Hash + +The cloud storage system supports various hash types of the objects. The +hashes are used when transferring data as an integrity check and can be +specifically used with the --checksum flag in syncs and in the check +command. + +To use the verify checksums when transferring between cloud storage +systems they must support a common hash type. + +† Note that Dropbox supports its own custom hash. This is an SHA256 sum +of all the 4MB block SHA256s. + +‡ SFTP supports checksums if the same login has shell access and md5sum +or sha1sum as well as echo are in the remote's PATH. + +†† WebDAV supports modtimes when used with Owncloud and Nextcloud only. + +‡‡ Microsoft OneDrive Personal supports SHA1 hashes, whereas OneDrive +for business and SharePoint server support Microsoft's own QuickXorHash. + +ModTime + +The cloud storage system supports setting modification times on objects. +If it does then this enables a using the modification times as part of +the sync. If not then only the size will be checked by default, though +the MD5SUM can be checked with the --checksum flag. + +All cloud storage systems support some kind of date on the object and +these will be set when transferring from the cloud storage system. + +Case Insensitive + +If a cloud storage systems is case sensitive then it is possible to have +two files which differ only in case, eg file.txt and FILE.txt. If a +cloud storage system is case insensitive then that isn't possible. + +This can cause problems when syncing between a case insensitive system +and a case sensitive system. The symptom of this is that no matter how +many times you run the sync it never completes fully. + +The local filesystem and SFTP may or may not be case sensitive depending +on OS. + +- Windows - usually case insensitive, though case is preserved +- OSX - usually case insensitive, though it is possible to format case + sensitive +- Linux - usually case sensitive, but there are case insensitive file + systems (eg FAT formatted USB keys) + +Most of the time this doesn't cause any problems as people tend to avoid +files whose name differs only by case even on case sensitive systems. + +Duplicate files + +If a cloud storage system allows duplicate files then it can have two +objects with the same name. + +This confuses rclone greatly when syncing - use the rclone dedupe +command to rename or remove duplicates. + +MIME Type + +MIME types (also known as media types) classify types of documents using +a simple text classification, eg text/html or application/pdf. + +Some cloud storage systems support reading (R) the MIME type of objects +and some support writing (W) the MIME type of objects. + +The MIME type can be important if you are serving files directly to HTTP +from the storage system. + +If you are copying from a remote which supports reading (R) to a remote +which supports writing (W) then rclone will preserve the MIME types. +Otherwise they will be guessed from the extension, or the remote itself +may assign the MIME type. + + +Optional Features + +All the remotes support a basic set of features, but there are some +optional features supported by some remotes used to make some operations +more efficient. + + Name Purge Copy Move DirMove CleanUp ListR StreamUpload LinkSharing About + ------------------------------ ------- ------ ------ --------- --------- ------- -------------- ------------- ------- + Amazon Drive Yes No Yes Yes No #575 No No No #2178 No + Amazon S3 No Yes No No No Yes Yes No #2178 No + Backblaze B2 No No No No Yes Yes Yes No #2178 No + Box Yes Yes Yes Yes No #575 No Yes No #2178 No + Dropbox Yes Yes Yes Yes No #575 No Yes Yes Yes + FTP No No Yes Yes No No Yes No #2178 No + Google Cloud Storage Yes Yes No No No Yes Yes No #2178 No + Google Drive Yes Yes Yes Yes Yes Yes Yes Yes Yes + HTTP No No No No No No No No #2178 No + Hubic Yes † Yes No No No Yes Yes No #2178 Yes + Jottacloud Yes Yes Yes Yes No No No No No + Mega Yes No Yes Yes No No No No #2178 Yes + Microsoft Azure Blob Storage Yes Yes No No No Yes No No #2178 No + Microsoft OneDrive Yes Yes Yes Yes No #575 No No No #2178 Yes + OpenDrive Yes Yes Yes Yes No No No No No + Openstack Swift Yes † Yes No No No Yes Yes No #2178 Yes + pCloud Yes Yes Yes Yes Yes No No No #2178 Yes + QingStor No Yes No No No Yes No No #2178 No + SFTP No No Yes Yes No No Yes No #2178 No + WebDAV Yes Yes Yes Yes No No Yes ‡ No #2178 No + Yandex Disk Yes No No No Yes Yes Yes No #2178 No + The local filesystem Yes No Yes Yes No No Yes No Yes + +Purge + +This deletes a directory quicker than just deleting all the files in the +directory. + +† Note Swift and Hubic implement this in order to delete directory +markers but they don't actually have a quicker way of deleting files +other than deleting them individually. + +‡ StreamUpload is not supported with Nextcloud + +Copy + +Used when copying an object to and from the same remote. This known as a +server side copy so you can copy a file without downloading it and +uploading it again. It is used if you use rclone copy or rclone move if +the remote doesn't support Move directly. + +If the server doesn't support Copy directly then for copy operations the +file is downloaded then re-uploaded. + +Move + +Used when moving/renaming an object on the same remote. This is known as +a server side move of a file. This is used in rclone move if the server +doesn't support DirMove. + +If the server isn't capable of Move then rclone simulates it with Copy +then delete. If the server doesn't support Copy then rclone will +download the file and re-upload it. + +DirMove + +This is used to implement rclone move to move a directory if possible. +If it isn't then it will use Move on each file (which falls back to Copy +then download and upload - see Move section). + +CleanUp + +This is used for emptying the trash for a remote by rclone cleanup. + +If the server can't do CleanUp then rclone cleanup will return an error. + +ListR + +The remote supports a recursive list to list all the contents beneath a +directory quickly. This enables the --fast-list flag to work. See the +rclone docs for more details. + +StreamUpload + +Some remotes allow files to be uploaded without knowing the file size in +advance. This allows certain operations to work without spooling the +file to local disk first, e.g. rclone rcat. + +LinkSharing + +Sets the necessary permissions on a file or folder and prints a link +that allows others to access them, even if they don't have an account on +the particular cloud provider. + +About + +This is used to fetch quota information from the remote, like bytes +used/free/quota and bytes used in the trash. + +If the server can't do About then rclone about will return an error. + + +Alias + +The alias remote provides a new name for another remote. + +Paths may be as deep as required or a local path, eg +remote:directory/subdirectory or /directory/subdirectory. + +During the initial setup with rclone config you will specify the target +remote. The target remote can either be a local path or another remote. + +Subfolders can be used in target remote. Asume a alias remote named +backup with the target mydrive:private/backup. Invoking +rclone mkdir backup:desktop is exactly the same as invoking +rclone mkdir mydrive:private/backup/desktop. + +There will be no special handling of paths containing .. segments. +Invoking rclone mkdir backup:../desktop is exactly the same as invoking +rclone mkdir mydrive:private/backup/../desktop. The empty path is not +allowed as a remote. To alias the current directory use . instead. + +Here is an example of how to make a alias called remote for local +folder. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" + 2 / Amazon Drive + \ "amazon cloud drive" + 3 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 4 / Backblaze B2 + \ "b2" + 5 / Box + \ "box" + 6 / Cache a remote + \ "cache" + 7 / Dropbox + \ "dropbox" + 8 / Encrypt/Decrypt a remote + \ "crypt" + 9 / FTP Connection + \ "ftp" + 10 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 11 / Google Drive + \ "drive" + 12 / Hubic + \ "hubic" + 13 / Local Disk + \ "local" + 14 / Microsoft Azure Blob Storage + \ "azureblob" + 15 / Microsoft OneDrive + \ "onedrive" + 16 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 17 / Pcloud + \ "pcloud" + 18 / QingCloud Object Storage + \ "qingstor" + 19 / SSH/SFTP Connection + \ "sftp" + 20 / Webdav + \ "webdav" + 21 / Yandex Disk + \ "yandex" + 22 / http Connection + \ "http" + Storage> 1 + Remote or path to alias. + Can be "myremote:path/to/dir", "myremote:bucket", "myremote:" or "/local/path". + remote> /mnt/storage/backup + Remote config + -------------------- + [remote] + remote = /mnt/storage/backup + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + Current remotes: + + Name Type + ==== ==== + remote alias + + e) Edit existing remote + n) New remote + d) Delete remote + r) Rename remote + c) Copy remote + s) Set configuration password + q) Quit config + e/n/d/r/c/s/q> q + +Once configured you can then use rclone like this, + +List directories in top level in /mnt/storage/backup + + rclone lsd remote: + +List all the files in /mnt/storage/backup + + rclone ls remote: + +Copy another local directory to the alias directory called source + + rclone copy /home/source remote:source + + +Amazon Drive + +Amazon Drive, formerly known as Amazon Cloud Drive, is a cloud storage +service run by Amazon for consumers. + + +Status + +IMPORTANT: rclone supports Amazon Drive only if you have your own set of +API keys. Unfortunately the Amazon Drive developer program is now closed +to new entries so if you don't already have your own set of keys you +will not be able to use rclone with Amazon Drive. + +For the history on why rclone no longer has a set of Amazon Drive API +keys see the forum. + +If you happen to know anyone who works at Amazon then please ask them to +re-instate rclone into the Amazon Drive developer program - thanks! + + +Setup + +The initial setup for Amazon Drive involves getting a token from Amazon +which you need to do in your browser. rclone config walks you through +it. + +The configuration process for Amazon Drive may involve using an oauth +proxy. This is used to keep the Amazon credentials out of the source +code. The proxy runs in Google's very secure App Engine environment and +doesn't store any credentials which pass through it. + +Since rclone doesn't currently have its own Amazon Drive credentials so +you will either need to have your own client_id and client_secret with +Amazon Drive, or use a a third party ouath proxy in which case you will +need to enter client_id, client_secret, auth_url and token_url. + +Note also if you are not using Amazon's auth_url and token_url, (ie you +filled in something for those) then if setting up on a remote machine +you can only use the copying the config method of configuration - +rclone authorize will not work. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + r) Rename remote + c) Copy remote + s) Set configuration password + q) Quit config + n/r/c/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / FTP Connection + \ "ftp" + 7 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 8 / Google Drive + \ "drive" + 9 / Hubic + \ "hubic" + 10 / Local Disk + \ "local" + 11 / Microsoft OneDrive + \ "onedrive" + 12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 13 / SSH/SFTP Connection + \ "sftp" + 14 / Yandex Disk + \ "yandex" + Storage> 1 + Amazon Application Client Id - required. + client_id> your client ID goes here + Amazon Application Client Secret - required. + client_secret> your client secret goes here + Auth server URL - leave blank to use Amazon's. + auth_url> Optional auth URL + Token server url - leave blank to use Amazon's. + token_url> Optional token URL + Remote config + Make sure your Redirect URL is set to "http://127.0.0.1:53682/" in your custom config. + Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine + y) Yes + n) No + y/n> y + If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth + Log in and authorize rclone for access + Waiting for code... + Got code + -------------------- + [remote] + client_id = your client ID goes here + client_secret = your client secret goes here + auth_url = Optional auth URL + token_url = Optional token URL + token = {"access_token":"xxxxxxxxxxxxxxxxxxxxxxx","token_type":"bearer","refresh_token":"xxxxxxxxxxxxxxxxxx","expiry":"2015-09-06T16:07:39.658438471+01:00"} + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Amazon. This only runs from the moment it opens +your browser to the moment you get back the verification code. This is +on http://127.0.0.1:53682/ and this it may require you to unblock it +temporarily if you are running a host firewall. + +Once configured you can then use rclone like this, + +List directories in top level of your Amazon Drive + + rclone lsd remote: + +List all the files in your Amazon Drive + + rclone ls remote: + +To copy a local directory to an Amazon Drive directory called backup + + rclone copy /home/source remote:backup + +Modified time and MD5SUMs + +Amazon Drive doesn't allow modification times to be changed via the API +so these won't be accurate or used for syncing. + +It does store MD5SUMs so for a more accurate sync, you can use the +--checksum flag. + +Deleting files + +Any files you delete with rclone will end up in the trash. Amazon don't +provide an API to permanently delete files, nor to empty the trash, so +you will have to do that with one of Amazon's apps or via the Amazon +Drive website. As of November 17, 2016, files are automatically deleted +by Amazon from the trash after 30 days. + +Using with non .com Amazon accounts + +Let's say you usually use amazon.co.uk. When you authenticate with +rclone it will take you to an amazon.com page to log in. Your +amazon.co.uk email and password should work here just fine. + +Specific options + +Here are the command line options specific to this cloud storage system. + +--acd-templink-threshold=SIZE + +Files this size or more will be downloaded via their tempLink. This is +to work around a problem with Amazon Drive which blocks downloads of +files bigger than about 10GB. The default for this is 9GB which +shouldn't need to be changed. + +To download files above this threshold, rclone requests a tempLink which +downloads the file through a temporary URL directly from the underlying +S3 storage. + +--acd-upload-wait-per-gb=TIME + +Sometimes Amazon Drive gives an error when a file has been fully +uploaded but the file appears anyway after a little while. This happens +sometimes for files over 1GB in size and nearly every time for files +bigger than 10GB. This parameter controls the time rclone waits for the +file to appear. + +The default value for this parameter is 3 minutes per GB, so by default +it will wait 3 minutes for every GB uploaded to see if the file appears. + +You can disable this feature by setting it to 0. This may cause conflict +errors as rclone retries the failed upload but the file will most likely +appear correctly eventually. + +These values were determined empirically by observing lots of uploads of +big files for a range of file sizes. + +Upload with the -v flag to see more info about what rclone is doing in +this situation. + +Limitations + +Note that Amazon Drive is case insensitive so you can't have a file +called "Hello.doc" and one called "hello.doc". + +Amazon Drive has rate limiting so you may notice errors in the sync (429 +errors). rclone will automatically retry the sync up to 3 times by +default (see --retries flag) which should hopefully work around this +problem. + +Amazon Drive has an internal limit of file sizes that can be uploaded to +the service. This limit is not officially published, but all files +larger than this will fail. + +At the time of writing (Jan 2016) is in the area of 50GB per file. This +means that larger files are likely to fail. + +Unfortunately there is no way for rclone to see that this failure is +because of file size, so it will retry the operation, as any other +failure. To avoid this problem, use --max-size 50000M option to limit +the maximum size of uploaded files. Note that --max-size does not split +files into segments, it only ignores files over this size. + + +Amazon S3 Storage Providers + +The S3 backend can be used with a number of different providers: + +- AWS S3 +- Ceph +- DigitalOcean Spaces +- Dreamhost +- IBM COS S3 +- Minio +- Wasabi + +Paths are specified as remote:bucket (or remote: for the lsd command.) +You may put subdirectories in too, eg remote:bucket/path/to/dir. + +Once you have made a remote (see the provider specific section above) +you can use it like this: + +See all buckets + + rclone lsd remote: + +Make a new bucket + + rclone mkdir remote:bucket + +List the contents of a bucket + + rclone ls remote:bucket + +Sync /home/local/directory to the remote bucket, deleting any excess +files in the bucket. + + rclone sync /home/local/directory remote:bucket + + +AWS S3 + +Here is an example of making an s3 configuration. First run + + rclone config + +This will guide you through an interactive setup process. + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" + 2 / Amazon Drive + \ "amazon cloud drive" + 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio) + \ "s3" + 4 / Backblaze B2 + \ "b2" + [snip] + 23 / http Connection + \ "http" + Storage> s3 + Choose your S3 provider. + Choose a number from below, or type in your own value + 1 / Amazon Web Services (AWS) S3 + \ "AWS" + 2 / Ceph Object Storage + \ "Ceph" + 3 / Digital Ocean Spaces + \ "DigitalOcean" + 4 / Dreamhost DreamObjects + \ "Dreamhost" + 5 / IBM COS S3 + \ "IBMCOS" + 6 / Minio Object Storage + \ "Minio" + 7 / Wasabi Object Storage + \ "Wasabi" + 8 / Any other S3 compatible provider + \ "Other" + provider> 1 + Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. + Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" + env_auth> 1 + AWS Access Key ID - leave blank for anonymous access or runtime credentials. + access_key_id> XXX + AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. + secret_access_key> YYY + Region to connect to. + Choose a number from below, or type in your own value + / The default endpoint - a good choice if you are unsure. + 1 | US Region, Northern Virginia or Pacific Northwest. + | Leave location constraint empty. + \ "us-east-1" + / US East (Ohio) Region + 2 | Needs location constraint us-east-2. + \ "us-east-2" + / US West (Oregon) Region + 3 | Needs location constraint us-west-2. + \ "us-west-2" + / US West (Northern California) Region + 4 | Needs location constraint us-west-1. + \ "us-west-1" + / Canada (Central) Region + 5 | Needs location constraint ca-central-1. + \ "ca-central-1" + / EU (Ireland) Region + 6 | Needs location constraint EU or eu-west-1. + \ "eu-west-1" + / EU (London) Region + 7 | Needs location constraint eu-west-2. + \ "eu-west-2" + / EU (Frankfurt) Region + 8 | Needs location constraint eu-central-1. + \ "eu-central-1" + / Asia Pacific (Singapore) Region + 9 | Needs location constraint ap-southeast-1. + \ "ap-southeast-1" + / Asia Pacific (Sydney) Region + 10 | Needs location constraint ap-southeast-2. + \ "ap-southeast-2" + / Asia Pacific (Tokyo) Region + 11 | Needs location constraint ap-northeast-1. + \ "ap-northeast-1" + / Asia Pacific (Seoul) + 12 | Needs location constraint ap-northeast-2. + \ "ap-northeast-2" + / Asia Pacific (Mumbai) + 13 | Needs location constraint ap-south-1. + \ "ap-south-1" + / South America (Sao Paulo) Region + 14 | Needs location constraint sa-east-1. + \ "sa-east-1" + region> 1 + Endpoint for S3 API. + Leave blank if using AWS to use the default endpoint for the region. + endpoint> + Location constraint - must be set to match the Region. Used when creating buckets only. + Choose a number from below, or type in your own value + 1 / Empty for US Region, Northern Virginia or Pacific Northwest. + \ "" + 2 / US East (Ohio) Region. + \ "us-east-2" + 3 / US West (Oregon) Region. + \ "us-west-2" + 4 / US West (Northern California) Region. + \ "us-west-1" + 5 / Canada (Central) Region. + \ "ca-central-1" + 6 / EU (Ireland) Region. + \ "eu-west-1" + 7 / EU (London) Region. + \ "eu-west-2" + 8 / EU Region. + \ "EU" + 9 / Asia Pacific (Singapore) Region. + \ "ap-southeast-1" + 10 / Asia Pacific (Sydney) Region. + \ "ap-southeast-2" + 11 / Asia Pacific (Tokyo) Region. + \ "ap-northeast-1" + 12 / Asia Pacific (Seoul) + \ "ap-northeast-2" + 13 / Asia Pacific (Mumbai) + \ "ap-south-1" + 14 / South America (Sao Paulo) Region. + \ "sa-east-1" + location_constraint> 1 + Canned ACL used when creating buckets and/or storing objects in S3. + For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" + 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access. + \ "public-read" + / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. + 3 | Granting this on a bucket is generally not recommended. + \ "public-read-write" + 4 / Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. + \ "authenticated-read" + / Object owner gets FULL_CONTROL. Bucket owner gets READ access. + 5 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ "bucket-owner-read" + / Both the object owner and the bucket owner get FULL_CONTROL over the object. + 6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ "bucket-owner-full-control" + acl> 1 + The server-side encryption algorithm used when storing this object in S3. + Choose a number from below, or type in your own value + 1 / None + \ "" + 2 / AES256 + \ "AES256" + server_side_encryption> 1 + The storage class to use when storing objects in S3. + Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Standard storage class + \ "STANDARD" + 3 / Reduced redundancy storage class + \ "REDUCED_REDUNDANCY" + 4 / Standard Infrequent Access storage class + \ "STANDARD_IA" + 5 / One Zone Infrequent Access storage class + \ "ONEZONE_IA" + storage_class> 1 + Remote config + -------------------- + [remote] + type = s3 + provider = AWS + env_auth = false + access_key_id = XXX + secret_access_key = YYY + region = us-east-1 + endpoint = + location_constraint = + acl = private + server_side_encryption = + storage_class = + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> + +--fast-list + +This remote supports --fast-list which allows you to use fewer +transactions in exchange for more memory. See the rclone docs for more +details. + +--update and --use-server-modtime + +As noted below, the modified time is stored on metadata on the object. +It is used by default for all operations that require checking the time +a file was last updated. It allows rclone to treat the remote more like +a true filesystem, but it is inefficient because it requires an extra +API call to retrieve the metadata. + +For many operations, the time the object was last uploaded to the remote +is sufficient to determine if it is "dirty". By using --update along +with --use-server-modtime, you can avoid the extra API call and simply +upload files whose local modtime is newer than the time it was last +uploaded. + +Modified time + +The modified time is stored as metadata on the object as +X-Amz-Meta-Mtime as floating point since the epoch accurate to 1 ns. + +Multipart uploads + +rclone supports multipart uploads with S3 which means that it can upload +files bigger than 5GB. Note that files uploaded _both_ with multipart +upload _and_ through crypt remotes do not have MD5 sums. + +Buckets and Regions + +With Amazon S3 you can list buckets (rclone lsd) using any region, but +you can only access the content of a bucket from the region it was +created in. If you attempt to access a bucket from the wrong region, you +will get an error, incorrect region, the bucket is not in 'XXX' region. + +Authentication + +There are a number of ways to supply rclone with a set of AWS +credentials, with and without using the environment. + +The different authentication methods are tried in this order: + +- Directly in the rclone configuration file (env_auth = false in the + config file): +- access_key_id and secret_access_key are required. +- session_token can be optionally set when using AWS STS. +- Runtime configuration (env_auth = true in the config file): +- Export the following environment variables before running rclone: + - Access Key ID: AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY + - Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY + - Session Token: AWS_SESSION_TOKEN (optional) +- Or, use a named profile: + - Profile files are standard files used by AWS CLI tools + - By default it will use the profile in your home directory (eg + ~/.aws/credentials on unix based systems) file and the "default" + profile, to change set these environment variables: + - AWS_SHARED_CREDENTIALS_FILE to control which file. + - AWS_PROFILE to control which profile to use. +- Or, run rclone in an ECS task with an IAM role (AWS only). +- Or, run rclone on an EC2 instance with an IAM role (AWS only). + +If none of these option actually end up providing rclone with AWS +credentials then S3 interaction will be non-authenticated (see below). + +S3 Permissions + +When using the sync subcommand of rclone the following minimum +permissions are required to be available on the bucket being written to: + +- ListBucket +- DeleteObject +- GetObject +- PutObject +- PutObjectACL + +Example policy: + + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::USER_SID:user/USER_NAME" + }, + "Action": [ + "s3:ListBucket", + "s3:DeleteObject", + "s3:GetObject", + "s3:PutObject", + "s3:PutObjectAcl" + ], + "Resource": [ + "arn:aws:s3:::BUCKET_NAME/*", + "arn:aws:s3:::BUCKET_NAME" + ] + } + ] + } + +Notes on above: + +1. This is a policy that can be used when creating bucket. It assumes + that USER_NAME has been created. +2. The Resource entry must include both resource ARNs, as one implies + the bucket and the other implies the bucket's objects. + +For reference, here's an Ansible script that will generate one or more +buckets that will work with rclone sync. + +Key Management System (KMS) + +If you are using server side encryption with KMS then you will find you +can't transfer small objects. As a work-around you can use the +--ignore-checksum flag. + +A proper fix is being worked on in issue #1824. + +Glacier + +You can transition objects to glacier storage using a lifecycle policy. +The bucket can still be synced or copied into normally, but if rclone +tries to access the data you will see an error like below. + + 2017/09/11 19:07:43 Failed to sync: failed to open source object: Object in GLACIER, restore first: path/to/file + +In this case you need to restore the object(s) in question before using +rclone. + +Specific options + +Here are the command line options specific to this cloud storage system. + +--s3-acl=STRING + +Canned ACL used when creating buckets and/or storing objects in S3. + +For more info visit the canned ACL docs. + +--s3-storage-class=STRING + +Storage class to upload new objects with. + +Available options include: + +- STANDARD - default storage class +- STANDARD_IA - for less frequently accessed data (e.g backups) +- ONEZONE_IA - for storing data in only one Availability Zone +- REDUCED_REDUNDANCY (only for noncritical, reproducible data, has + lower redundancy) + +--s3-chunk-size=SIZE + +Any files larger than this will be uploaded in chunks of this size. The +default is 5MB. The minimum is 5MB. + +Note that 2 chunks of this size are buffered in memory per transfer. + +If you are transferring large files over high speed links and you have +enough memory, then increasing this will speed up the transfers. + +--s3-force-path-style=BOOL + +If this is true (the default) then rclone will use path style access, if +false then rclone will use virtual path style. See the AWS S3 docs for +more info. + +Some providers (eg Aliyun OSS or Netease COS) require this set to false. +It can also be set in the config in the advanced section. + +--s3-upload-concurrency + +Number of chunks of the same file that are uploaded concurrently. +Default is 2. + +If you are uploading small amount of large file over high speed link and +these uploads do not fully utilize your bandwidth, then increasing this +may help to speed up the transfers. + +Anonymous access to public buckets + +If you want to use rclone to access a public bucket, configure with a +blank access_key_id and secret_access_key. Your config should end up +looking like this: + + [anons3] + type = s3 + provider = AWS + env_auth = false + access_key_id = + secret_access_key = + region = us-east-1 + endpoint = + location_constraint = + acl = private + server_side_encryption = + storage_class = + +Then use it as normal with the name of the public bucket, eg + + rclone lsd anons3:1000genomes + +You will be able to list and copy data but not upload it. + +Ceph + +Ceph is an open source unified, distributed storage system designed for +excellent performance, reliability and scalability. It has an S3 +compatible object storage interface. + +To use rclone with Ceph, configure as above but leave the region blank +and set the endpoint. You should end up with something like this in your +config: + + [ceph] + type = s3 + provider = Ceph + env_auth = false + access_key_id = XXX + secret_access_key = YYY + region = + endpoint = https://ceph.endpoint.example.com + location_constraint = + acl = + server_side_encryption = + storage_class = + +Note also that Ceph sometimes puts / in the passwords it gives users. If +you read the secret access key using the command line tools you will get +a JSON blob with the / escaped as \/. Make sure you only write / in the +secret access key. + +Eg the dump from Ceph looks something like this (irrelevant keys +removed). + + { + "user_id": "xxx", + "display_name": "xxxx", + "keys": [ + { + "user": "xxx", + "access_key": "xxxxxx", + "secret_key": "xxxxxx\/xxxx" + } + ], + } + +Because this is a json dump, it is encoding the / as \/, so if you use +the secret key as xxxxxx/xxxx it will work fine. + +Dreamhost + +Dreamhost DreamObjects is an object storage system based on CEPH. + +To use rclone with Dreamhost, configure as above but leave the region +blank and set the endpoint. You should end up with something like this +in your config: + + [dreamobjects] + type = s3 + provider = DreamHost + env_auth = false + access_key_id = your_access_key + secret_access_key = your_secret_key + region = + endpoint = objects-us-west-1.dream.io + location_constraint = + acl = private + server_side_encryption = + storage_class = + +DigitalOcean Spaces + +Spaces is an S3-interoperable object storage service from cloud provider +DigitalOcean. + +To connect to DigitalOcean Spaces you will need an access key and secret +key. These can be retrieved on the "Applications & API" page of the +DigitalOcean control panel. They will be needed when promted by +rclone config for your access_key_id and secret_access_key. + +When prompted for a region or location_constraint, press enter to use +the default value. The region must be included in the endpoint setting +(e.g. nyc3.digitaloceanspaces.com). The defualt values can be used for +other settings. + +Going through the whole process of creating a new remote by running +rclone config, each prompt should be answered as shown below: + + Storage> s3 + env_auth> 1 + access_key_id> YOUR_ACCESS_KEY + secret_access_key> YOUR_SECRET_KEY + region> + endpoint> nyc3.digitaloceanspaces.com + location_constraint> + acl> + storage_class> + +The resulting configuration file should look like: + + [spaces] + type = s3 + provider = DigitalOcean + env_auth = false + access_key_id = YOUR_ACCESS_KEY + secret_access_key = YOUR_SECRET_KEY + region = + endpoint = nyc3.digitaloceanspaces.com + location_constraint = + acl = + server_side_encryption = + storage_class = + +Once configured, you can create a new Space and begin copying files. For +example: + + rclone mkdir spaces:my-new-space + rclone copy /path/to/files spaces:my-new-space + +IBM COS (S3) + +Information stored with IBM Cloud Object Storage is encrypted and +dispersed across multiple geographic locations, and accessed through an +implementation of the S3 API. This service makes use of the distributed +storage technologies provided by IBM’s Cloud Object Storage System +(formerly Cleversafe). For more information visit: +(http://www.ibm.com/cloud/object-storage) + +To configure access to IBM COS S3, follow the steps below: + +1. Run rclone config and select n for a new remote. + + 2018/02/14 14:13:11 NOTICE: Config file "C:\\Users\\a\\.config\\rclone\\rclone.conf" not found - using defaults + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + +2. Enter the name for the configuration + + name> + +3. Select "s3" storage. + + Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" + 2 / Amazon Drive + \ "amazon cloud drive" + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, Minio, IBM COS) + \ "s3" + 4 / Backblaze B2 + \ "b2" + [snip] + 23 / http Connection + \ "http" + Storage> 3 + +4. Select IBM COS as the S3 Storage Provider. + + Choose the S3 provider. + Choose a number from below, or type in your own value + 1 / Choose this option to configure Storage to AWS S3 + \ "AWS" + 2 / Choose this option to configure Storage to Ceph Systems + \ "Ceph" + 3 / Choose this option to configure Storage to Dreamhost + \ "Dreamhost" + 4 / Choose this option to the configure Storage to IBM COS S3 + \ "IBMCOS" + 5 / Choose this option to the configure Storage to Minio + \ "Minio" + Provider>4 + +5. Enter the Access Key and Secret. + + AWS Access Key ID - leave blank for anonymous access or runtime credentials. + access_key_id> <> + AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. + secret_access_key> <> + +6. Specify the endpoint for IBM COS. For Public IBM COS, choose from + the option below. For On Premise IBM COS, enter an enpoint address. + + Endpoint for IBM COS S3 API. + Specify if using an IBM COS On Premise. + Choose a number from below, or type in your own value + 1 / US Cross Region Endpoint + \ "s3-api.us-geo.objectstorage.softlayer.net" + 2 / US Cross Region Dallas Endpoint + \ "s3-api.dal.us-geo.objectstorage.softlayer.net" + 3 / US Cross Region Washington DC Endpoint + \ "s3-api.wdc-us-geo.objectstorage.softlayer.net" + 4 / US Cross Region San Jose Endpoint + \ "s3-api.sjc-us-geo.objectstorage.softlayer.net" + 5 / US Cross Region Private Endpoint + \ "s3-api.us-geo.objectstorage.service.networklayer.com" + 6 / US Cross Region Dallas Private Endpoint + \ "s3-api.dal-us-geo.objectstorage.service.networklayer.com" + 7 / US Cross Region Washington DC Private Endpoint + \ "s3-api.wdc-us-geo.objectstorage.service.networklayer.com" + 8 / US Cross Region San Jose Private Endpoint + \ "s3-api.sjc-us-geo.objectstorage.service.networklayer.com" + 9 / US Region East Endpoint + \ "s3.us-east.objectstorage.softlayer.net" + 10 / US Region East Private Endpoint + \ "s3.us-east.objectstorage.service.networklayer.com" + 11 / US Region South Endpoint + [snip] + 34 / Toronto Single Site Private Endpoint + \ "s3.tor01.objectstorage.service.networklayer.com" + endpoint>1 + +7. Specify a IBM COS Location Constraint. The location constraint must + match endpoint when using IBM Cloud Public. For on-prem COS, do not + make a selection from this list, hit enter + + 1 / US Cross Region Standard + \ "us-standard" + 2 / US Cross Region Vault + \ "us-vault" + 3 / US Cross Region Cold + \ "us-cold" + 4 / US Cross Region Flex + \ "us-flex" + 5 / US East Region Standard + \ "us-east-standard" + 6 / US East Region Vault + \ "us-east-vault" + 7 / US East Region Cold + \ "us-east-cold" + 8 / US East Region Flex + \ "us-east-flex" + 9 / US South Region Standard + \ "us-south-standard" + 10 / US South Region Vault + \ "us-south-vault" + [snip] + 32 / Toronto Flex + \ "tor01-flex" + location_constraint>1 + +8. Specify a canned ACL. IBM Cloud (Strorage) supports "public-read" + and "private". IBM Cloud(Infra) supports all the canned ACLs. + On-Premise COS supports all the canned ACLs. + + Canned ACL used when creating buckets and/or storing objects in S3. + For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise COS + \ "private" + 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access. This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise IBM COS + \ "public-read" + 3 / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. This acl is available on IBM Cloud (Infra), On-Premise IBM COS + \ "public-read-write" + 4 / Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS + \ "authenticated-read" + acl> 1 + +9. Review the displayed configuration and accept to save the "remote" + then quit. The config file should look like this + + [xxx] + type = s3 + Provider = IBMCOS + access_key_id = xxx + secret_access_key = yyy + endpoint = s3-api.us-geo.objectstorage.softlayer.net + location_constraint = us-standard + acl = private + +10. Execute rclone commands + + 1) Create a bucket. + rclone mkdir IBM-COS-XREGION:newbucket + 2) List available buckets. + rclone lsd IBM-COS-XREGION: + -1 2017-11-08 21:16:22 -1 test + -1 2018-02-14 20:16:39 -1 newbucket + 3) List contents of a bucket. + rclone ls IBM-COS-XREGION:newbucket + 18685952 test.exe + 4) Copy a file from local to remote. + rclone copy /Users/file.txt IBM-COS-XREGION:newbucket + 5) Copy a file from remote to local. + rclone copy IBM-COS-XREGION:newbucket/file.txt . + 6) Delete a file on remote. + rclone delete IBM-COS-XREGION:newbucket/file.txt + +Minio + +Minio is an object storage server built for cloud application developers +and devops. + +It is very easy to install and provides an S3 compatible server which +can be used by rclone. + +To use it, install Minio following the instructions here. + +When it configures itself Minio will print something like this + + Endpoint: http://192.168.1.106:9000 http://172.23.0.1:9000 + AccessKey: USWUXHGYZQYFYFFIT3RE + SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 + Region: us-east-1 + SQS ARNs: arn:minio:sqs:us-east-1:1:redis arn:minio:sqs:us-east-1:2:redis + + Browser Access: + http://192.168.1.106:9000 http://172.23.0.1:9000 + + Command-line Access: https://docs.minio.io/docs/minio-client-quickstart-guide + $ mc config host add myminio http://192.168.1.106:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 + + Object API (Amazon S3 compatible): + Go: https://docs.minio.io/docs/golang-client-quickstart-guide + Java: https://docs.minio.io/docs/java-client-quickstart-guide + Python: https://docs.minio.io/docs/python-client-quickstart-guide + JavaScript: https://docs.minio.io/docs/javascript-client-quickstart-guide + .NET: https://docs.minio.io/docs/dotnet-client-quickstart-guide + + Drive Capacity: 26 GiB Free, 165 GiB Total + +These details need to go into rclone config like this. Note that it is +important to put the region in as stated above. + + env_auth> 1 + access_key_id> USWUXHGYZQYFYFFIT3RE + secret_access_key> MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 + region> us-east-1 + endpoint> http://192.168.1.106:9000 + location_constraint> + server_side_encryption> + +Which makes the config file look like this + + [minio] + type = s3 + provider = Minio + env_auth = false + access_key_id = USWUXHGYZQYFYFFIT3RE + secret_access_key = MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 + region = us-east-1 + endpoint = http://192.168.1.106:9000 + location_constraint = + server_side_encryption = + +So once set up, for example to copy files into a bucket + + rclone copy /path/to/files minio:bucket + +Wasabi + +Wasabi is a cloud-based object storage service for a broad range of +applications and use cases. Wasabi is designed for individuals and +organizations that require a high-performance, reliable, and secure data +storage infrastructure at minimal cost. + +Wasabi provides an S3 interface which can be configured for use with +rclone like this. + + No remotes found - make a new one + n) New remote + s) Set configuration password + n/s> n + name> wasabi + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + [snip] + Storage> s3 + Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. + Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" + env_auth> 1 + AWS Access Key ID - leave blank for anonymous access or runtime credentials. + access_key_id> YOURACCESSKEY + AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. + secret_access_key> YOURSECRETACCESSKEY + Region to connect to. + Choose a number from below, or type in your own value + / The default endpoint - a good choice if you are unsure. + 1 | US Region, Northern Virginia or Pacific Northwest. + | Leave location constraint empty. + \ "us-east-1" + [snip] + region> us-east-1 + Endpoint for S3 API. + Leave blank if using AWS to use the default endpoint for the region. + Specify if using an S3 clone such as Ceph. + endpoint> s3.wasabisys.com + Location constraint - must be set to match the Region. Used when creating buckets only. + Choose a number from below, or type in your own value + 1 / Empty for US Region, Northern Virginia or Pacific Northwest. + \ "" + [snip] + location_constraint> + Canned ACL used when creating buckets and/or storing objects in S3. + For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" + [snip] + acl> + The server-side encryption algorithm used when storing this object in S3. + Choose a number from below, or type in your own value + 1 / None + \ "" + 2 / AES256 + \ "AES256" + server_side_encryption> + The storage class to use when storing objects in S3. + Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Standard storage class + \ "STANDARD" + 3 / Reduced redundancy storage class + \ "REDUCED_REDUNDANCY" + 4 / Standard Infrequent Access storage class + \ "STANDARD_IA" + storage_class> + Remote config + -------------------- + [wasabi] + env_auth = false + access_key_id = YOURACCESSKEY + secret_access_key = YOURSECRETACCESSKEY + region = us-east-1 + endpoint = s3.wasabisys.com + location_constraint = + acl = + server_side_encryption = + storage_class = + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +This will leave the config file looking like this. + + [wasabi] + type = s3 + provider = Wasabi + env_auth = false + access_key_id = YOURACCESSKEY + secret_access_key = YOURSECRETACCESSKEY + region = + endpoint = s3.wasabisys.com + location_constraint = + acl = + server_side_encryption = + storage_class = + +Aliyun OSS / Netease NOS + +This describes how to set up Aliyun OSS - Netease NOS is the same except +for different endpoints. + +Note this is a pretty standard S3 setup, except for the setting of +force_path_style = false in the advanced config. + + # rclone config + e/n/d/r/c/s/q> n + name> oss + Type of storage to configure. + Enter a string value. Press Enter for the default (""). + Choose a number from below, or type in your own value + 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio) + \ "s3" + Storage> s3 + Choose your S3 provider. + Enter a string value. Press Enter for the default (""). + Choose a number from below, or type in your own value + 8 / Any other S3 compatible provider + \ "Other" + provider> other + Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). + Only applies if access_key_id and secret_access_key is blank. + Enter a boolean value (true or false). Press Enter for the default ("false"). + Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" + env_auth> 1 + AWS Access Key ID. + Leave blank for anonymous access or runtime credentials. + Enter a string value. Press Enter for the default (""). + access_key_id> xxxxxxxxxxxx + AWS Secret Access Key (password) + Leave blank for anonymous access or runtime credentials. + Enter a string value. Press Enter for the default (""). + secret_access_key> xxxxxxxxxxxxxxxxx + Region to connect to. + Leave blank if you are using an S3 clone and you don't have a region. + Enter a string value. Press Enter for the default (""). + Choose a number from below, or type in your own value + 1 / Use this if unsure. Will use v4 signatures and an empty region. + \ "" + 2 / Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH. + \ "other-v2-signature" + region> 1 + Endpoint for S3 API. + Required when using an S3 clone. + Enter a string value. Press Enter for the default (""). + Choose a number from below, or type in your own value + endpoint> oss-cn-shenzhen.aliyuncs.com + Location constraint - must be set to match the Region. + Leave blank if not sure. Used when creating buckets only. + Enter a string value. Press Enter for the default (""). + location_constraint> + Canned ACL used when creating buckets and/or storing objects in S3. + For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + Enter a string value. Press Enter for the default (""). + Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" + acl> 1 + Edit advanced config? (y/n) + y) Yes + n) No + y/n> y + Chunk size to use for uploading + Enter a size with suffix k,M,G,T. Press Enter for the default ("5M"). + chunk_size> + Don't store MD5 checksum with object metadata + Enter a boolean value (true or false). Press Enter for the default ("false"). + disable_checksum> + An AWS session token + Enter a string value. Press Enter for the default (""). + session_token> + Concurrency for multipart uploads. + Enter a signed integer. Press Enter for the default ("2"). + upload_concurrency> + If true use path style access if false use virtual hosted style. + Some providers (eg Aliyun OSS or Netease COS) require this. + Enter a boolean value (true or false). Press Enter for the default ("true"). + force_path_style> false + Remote config + -------------------- + [oss] + type = s3 + provider = Other + env_auth = false + access_key_id = xxxxxxxxx + secret_access_key = xxxxxxxxxxxxx + endpoint = oss-cn-shenzhen.aliyuncs.com + acl = private + force_path_style = false + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + + +Backblaze B2 + +B2 is Backblaze's cloud storage system. + +Paths are specified as remote:bucket (or remote: for the lsd command.) +You may put subdirectories in too, eg remote:bucket/path/to/dir. + +Here is an example of making a b2 configuration. First run + + rclone config + +This will guide you through an interactive setup process. You will need +your account number (a short hex number) and key (a long hex number) +which you can get from the b2 control panel. + + No remotes found - make a new one + n) New remote + q) Quit config + n/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" + 10 / Microsoft OneDrive + \ "onedrive" + 11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 12 / SSH/SFTP Connection + \ "sftp" + 13 / Yandex Disk + \ "yandex" + Storage> 3 + Account ID or Application Key ID + account> 123456789abc + Application Key + key> 0123456789abcdef0123456789abcdef0123456789 + Endpoint for the service - leave blank normally. + endpoint> + Remote config + -------------------- + [remote] + account = 123456789abc + key = 0123456789abcdef0123456789abcdef0123456789 + endpoint = + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +This remote is called remote and can now be used like this + +See all buckets + + rclone lsd remote: + +Create a new bucket + + rclone mkdir remote:bucket + +List the contents of a bucket + + rclone ls remote:bucket + +Sync /home/local/directory to the remote bucket, deleting any excess +files in the bucket. + + rclone sync /home/local/directory remote:bucket + +Application Keys + +B2 supports multiple Application Keys for different access permission to +B2 Buckets. + +You can use these with rclone too. + +Follow Backblaze's docs to create an Application Key with the required +permission and add the Application Key ID as the account and the +Application Key itself as the key. + +Note that you must put the Application Key ID as the account - you can't +use the master Account ID. If you try then B2 will return 401 errors. + +--fast-list + +This remote supports --fast-list which allows you to use fewer +transactions in exchange for more memory. See the rclone docs for more +details. + +Modified time + +The modified time is stored as metadata on the object as +X-Bz-Info-src_last_modified_millis as milliseconds since 1970-01-01 in +the Backblaze standard. Other tools should be able to use this as a +modified time. + +Modified times are used in syncing and are fully supported except in the +case of updating a modification time on an existing object. In this case +the object will be uploaded again as B2 doesn't have an API method to +set the modification time independent of doing an upload. + +SHA1 checksums + +The SHA1 checksums of the files are checked on upload and download and +will be used in the syncing process. + +Large files (bigger than the limit in --b2-upload-cutoff) which are +uploaded in chunks will store their SHA1 on the object as +X-Bz-Info-large_file_sha1 as recommended by Backblaze. + +For a large file to be uploaded with an SHA1 checksum, the source needs +to support SHA1 checksums. The local disk supports SHA1 checksums so +large file transfers from local disk will have an SHA1. See the overview +for exactly which remotes support SHA1. + +Sources which don't support SHA1, in particular crypt will upload large +files without SHA1 checksums. This may be fixed in the future (see +#1767). + +Files sizes below --b2-upload-cutoff will always have an SHA1 regardless +of the source. + +Transfers + +Backblaze recommends that you do lots of transfers simultaneously for +maximum speed. In tests from my SSD equipped laptop the optimum setting +is about --transfers 32 though higher numbers may be used for a slight +speed improvement. The optimum number for you may vary depending on your +hardware, how big the files are, how much you want to load your +computer, etc. The default of --transfers 4 is definitely too low for +Backblaze B2 though. + +Note that uploading big files (bigger than 200 MB by default) will use a +96 MB RAM buffer by default. There can be at most --transfers of these +in use at any moment, so this sets the upper limit on the memory used. + +Versions + +When rclone uploads a new version of a file it creates a new version of +it. Likewise when you delete a file, the old version will be marked +hidden and still be available. Conversely, you may opt in to a "hard +delete" of files with the --b2-hard-delete flag which would permanently +remove the file instead of hiding it. + +Old versions of files, where available, are visible using the +--b2-versions flag. + +If you wish to remove all the old versions then you can use the +rclone cleanup remote:bucket command which will delete all the old +versions of files, leaving the current ones intact. You can also supply +a path and only old versions under that path will be deleted, eg +rclone cleanup remote:bucket/path/to/stuff. + +When you purge a bucket, the current and the old versions will be +deleted then the bucket will be deleted. + +However delete will cause the current versions of the files to become +hidden old versions. + +Here is a session showing the listing and retrieval of an old version +followed by a cleanup of the old versions. + +Show current version and all the versions with --b2-versions flag. + + $ rclone -q ls b2:cleanup-test + 9 one.txt + + $ rclone -q --b2-versions ls b2:cleanup-test + 9 one.txt + 8 one-v2016-07-04-141032-000.txt + 16 one-v2016-07-04-141003-000.txt + 15 one-v2016-07-02-155621-000.txt + +Retrieve an old version + + $ rclone -q --b2-versions copy b2:cleanup-test/one-v2016-07-04-141003-000.txt /tmp + + $ ls -l /tmp/one-v2016-07-04-141003-000.txt + -rw-rw-r-- 1 ncw ncw 16 Jul 2 17:46 /tmp/one-v2016-07-04-141003-000.txt + +Clean up all the old versions and show that they've gone. + + $ rclone -q cleanup b2:cleanup-test + + $ rclone -q ls b2:cleanup-test + 9 one.txt + + $ rclone -q --b2-versions ls b2:cleanup-test + 9 one.txt + +Data usage + +It is useful to know how many requests are sent to the server in +different scenarios. + +All copy commands send the following 4 requests: + + /b2api/v1/b2_authorize_account + /b2api/v1/b2_create_bucket + /b2api/v1/b2_list_buckets + /b2api/v1/b2_list_file_names + +The b2_list_file_names request will be sent once for every 1k files in +the remote path, providing the checksum and modification time of the +listed files. As of version 1.33 issue #818 causes extra requests to be +sent when using B2 with Crypt. When a copy operation does not require +any files to be uploaded, no more requests will be sent. + +Uploading files that do not require chunking, will send 2 requests per +file upload: + + /b2api/v1/b2_get_upload_url + /b2api/v1/b2_upload_file/ + +Uploading files requiring chunking, will send 2 requests (one each to +start and finish the upload) and another 2 requests for each chunk: + + /b2api/v1/b2_start_large_file + /b2api/v1/b2_get_upload_part_url + /b2api/v1/b2_upload_part/ + /b2api/v1/b2_finish_large_file + +Specific options + +Here are the command line options specific to this cloud storage system. + +--b2-chunk-size valuee=SIZE + +When uploading large files chunk the file into this size. Note that +these chunks are buffered in memory and there might a maximum of +--transfers chunks in progress at once. 5,000,000 Bytes is the minimim +size (default 96M). + +--b2-upload-cutoff=SIZE + +Cutoff for switching to chunked upload (default 190.735 MiB == 200 MB). +Files above this size will be uploaded in chunks of --b2-chunk-size. + +This value should be set no larger than 4.657GiB (== 5GB) as this is the +largest file size that can be uploaded. + +--b2-test-mode=FLAG + +This is for debugging purposes only. + +Setting FLAG to one of the strings below will cause b2 to return +specific errors for debugging purposes. + +- fail_some_uploads +- expire_some_account_authorization_tokens +- force_cap_exceeded + +These will be set in the X-Bz-Test-Mode header which is documented in +the b2 integrations checklist. + +--b2-versions + +When set rclone will show and act on older versions of files. For +example + +Listing without --b2-versions + + $ rclone -q ls b2:cleanup-test + 9 one.txt + +And with + + $ rclone -q --b2-versions ls b2:cleanup-test + 9 one.txt + 8 one-v2016-07-04-141032-000.txt + 16 one-v2016-07-04-141003-000.txt + 15 one-v2016-07-02-155621-000.txt + +Showing that the current version is unchanged but older versions can be +seen. These have the UTC date that they were uploaded to the server to +the nearest millisecond appended to them. + +Note that when using --b2-versions no file write operations are +permitted, so you can't upload files or delete them. + + +Box + +Paths are specified as remote:path + +Paths may be as deep as required, eg remote:directory/subdirectory. + +The initial setup for Box involves getting a token from Box which you +need to do in your browser. rclone config walks you through it. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Box + \ "box" + 5 / Dropbox + \ "dropbox" + 6 / Encrypt/Decrypt a remote + \ "crypt" + 7 / FTP Connection + \ "ftp" + 8 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 9 / Google Drive + \ "drive" + 10 / Hubic + \ "hubic" + 11 / Local Disk + \ "local" + 12 / Microsoft OneDrive + \ "onedrive" + 13 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 14 / SSH/SFTP Connection + \ "sftp" + 15 / Yandex Disk + \ "yandex" + 16 / http Connection + \ "http" + Storage> box + Box App Client Id - leave blank normally. + client_id> + Box App Client Secret - leave blank normally. + client_secret> + Remote config + Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine + y) Yes + n) No + y/n> y + If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth + Log in and authorize rclone for access + Waiting for code... + Got code + -------------------- + [remote] + client_id = + client_secret = + token = {"access_token":"XXX","token_type":"bearer","refresh_token":"XXX","expiry":"XXX"} + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Box. This only runs from the moment it opens your +browser to the moment you get back the verification code. This is on +http://127.0.0.1:53682/ and this it may require you to unblock it +temporarily if you are running a host firewall. + +Once configured you can then use rclone like this, + +List directories in top level of your Box + + rclone lsd remote: + +List all the files in your Box + + rclone ls remote: + +To copy a local directory to an Box directory called backup + + rclone copy /home/source remote:backup + +Invalid refresh token + +According to the box docs: + + Each refresh_token is valid for one use in 60 days. + +This means that if you + +- Don't use the box remote for 60 days +- Copy the config file with a box refresh token in and use it in two + places +- Get an error on a token refresh + +then rclone will return an error which includes the text +Invalid refresh token. + +To fix this you will need to use oauth2 again to update the refresh +token. You can use the methods in the remote setup docs, bearing in mind +that if you use the copy the config file method, you should not use that +remote on the computer you did the authentication on. + +Here is how to do it. + + $ rclone config + Current remotes: + + Name Type + ==== ==== + remote box + + e) Edit existing remote + n) New remote + d) Delete remote + r) Rename remote + c) Copy remote + s) Set configuration password + q) Quit config + e/n/d/r/c/s/q> e + Choose a number from below, or type in an existing value + 1 > remote + remote> remote + -------------------- + [remote] + type = box + token = {"access_token":"XXX","token_type":"bearer","refresh_token":"XXX","expiry":"2017-07-08T23:40:08.059167677+01:00"} + -------------------- + Edit remote + Value "client_id" = "" + Edit? (y/n)> + y) Yes + n) No + y/n> n + Value "client_secret" = "" + Edit? (y/n)> + y) Yes + n) No + y/n> n + Remote config + Already have a token - refresh? + y) Yes + n) No + y/n> y + Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine + y) Yes + n) No + y/n> y + If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth + Log in and authorize rclone for access + Waiting for code... + Got code + -------------------- + [remote] + type = box + token = {"access_token":"YYY","token_type":"bearer","refresh_token":"YYY","expiry":"2017-07-23T12:22:29.259137901+01:00"} + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +Modified time and hashes + +Box allows modification times to be set on objects accurate to 1 second. +These will be used to detect whether objects need syncing or not. + +Box supports SHA1 type hashes, so you can use the --checksum flag. + +Transfers + +For files above 50MB rclone will use a chunked transfer. Rclone will +upload up to --transfers chunks at the same time (shared among all the +multipart uploads). Chunks are buffered in memory and are normally 8MB +so increasing --transfers will increase memory use. + +Deleting files + +Depending on the enterprise settings for your user, the item will either +be actually deleted from Box or moved to the trash. + +Specific options + +Here are the command line options specific to this cloud storage system. + +--box-upload-cutoff=SIZE + +Cutoff for switching to chunked upload - must be >= 50MB. The default is +50MB. + +--box-commit-retries int + +Max number of times to try committing a multipart file. (default 100) + +Limitations + +Note that Box is case insensitive so you can't have a file called +"Hello.doc" and one called "hello.doc". + +Box file names can't have the \ character in. rclone maps this to and +from an identical looking unicode equivalent ＼. + +Box only supports filenames up to 255 characters in length. + + +Cache (BETA) + +The cache remote wraps another existing remote and stores file structure +and its data for long running tasks like rclone mount. + +To get started you just need to have an existing remote which can be +configured with cache. + +Here is an example of how to make a remote called test-cache. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + r) Rename remote + c) Copy remote + s) Set configuration password + q) Quit config + n/r/c/s/q> n + name> test-cache + Type of storage to configure. + Choose a number from below, or type in your own value + ... + 5 / Cache a remote + \ "cache" + ... + Storage> 5 + Remote to cache. + Normally should contain a ':' and a path, eg "myremote:path/to/dir", + "myremote:bucket" or maybe "myremote:" (not recommended). + remote> local:/test + Optional: The URL of the Plex server + plex_url> http://127.0.0.1:32400 + Optional: The username of the Plex user + plex_username> dummyusername + Optional: The password of the Plex user + y) Yes type in my own password + g) Generate random password + n) No leave this optional password blank + y/g/n> y + Enter the password: + password: + Confirm the password: + password: + The size of a chunk. Lower value good for slow connections but can affect seamless reading. + Default: 5M + Choose a number from below, or type in your own value + 1 / 1MB + \ "1m" + 2 / 5 MB + \ "5M" + 3 / 10 MB + \ "10M" + chunk_size> 2 + How much time should object info (file size, file hashes etc) be stored in cache. Use a very high value if you don't plan on changing the source FS from outside the cache. + Accepted units are: "s", "m", "h". + Default: 5m + Choose a number from below, or type in your own value + 1 / 1 hour + \ "1h" + 2 / 24 hours + \ "24h" + 3 / 24 hours + \ "48h" + info_age> 2 + The maximum size of stored chunks. When the storage grows beyond this size, the oldest chunks will be deleted. + Default: 10G + Choose a number from below, or type in your own value + 1 / 500 MB + \ "500M" + 2 / 1 GB + \ "1G" + 3 / 10 GB + \ "10G" + chunk_total_size> 3 + Remote config + -------------------- + [test-cache] + remote = local:/test + plex_url = http://127.0.0.1:32400 + plex_username = dummyusername + plex_password = *** ENCRYPTED *** + chunk_size = 5M + info_age = 48h + chunk_total_size = 10G + +You can then use it like this, + +List directories in top level of your drive + + rclone lsd test-cache: + +List all the files in your drive + + rclone ls test-cache: + +To start a cached mount + + rclone mount --allow-other test-cache: /var/tmp/test-cache + +Write Features + +Offline uploading + +In an effort to make writing through cache more reliable, the backend +now supports this feature which can be activated by specifying a +cache-tmp-upload-path. + +A files goes through these states when using this feature: + +1. An upload is started (usually by copying a file on the cache remote) +2. When the copy to the temporary location is complete the file is part + of the cached remote and looks and behaves like any other file + (reading included) +3. After cache-tmp-wait-time passes and the file is next in line, + rclone move is used to move the file to the cloud provider +4. Reading the file still works during the upload but most + modifications on it will be prohibited +5. Once the move is complete the file is unlocked for modifications as + it becomes as any other regular file +6. If the file is being read through cache when it's actually deleted + from the temporary path then cache will simply swap the source to + the cloud provider without interrupting the reading (small blip can + happen though) + +Files are uploaded in sequence and only one file is uploaded at a time. +Uploads will be stored in a queue and be processed based on the order +they were added. The queue and the temporary storage is persistent +across restarts and even purges of the cache. + +Write Support + +Writes are supported through cache. One caveat is that a mounted cache +remote does not add any retry or fallback mechanism to the upload +operation. This will depend on the implementation of the wrapped remote. +Consider using Offline uploading for reliable writes. + +One special case is covered with cache-writes which will cache the file +data at the same time as the upload when it is enabled making it +available from the cache store immediately once the upload is finished. + +Read Features + +Multiple connections + +To counter the high latency between a local PC where rclone is running +and cloud providers, the cache remote can split multiple requests to the +cloud provider for smaller file chunks and combines them together +locally where they can be available almost immediately before the reader +usually needs them. + +This is similar to buffering when media files are played online. Rclone +will stay around the current marker but always try its best to stay +ahead and prepare the data before. + +Plex Integration + +There is a direct integration with Plex which allows cache to detect +during reading if the file is in playback or not. This helps cache to +adapt how it queries the cloud provider depending on what is needed for. + +Scans will have a minimum amount of workers (1) while in a confirmed +playback cache will deploy the configured number of workers. + +This integration opens the doorway to additional performance +improvements which will be explored in the near future. + +NOTE: If Plex options are not configured, cache will function with its +configured options without adapting any of its settings. + +How to enable? Run rclone config and add all the Plex options (endpoint, +username and password) in your remote and it will be automatically +enabled. + +Affected settings: - cache-workers: _Configured value_ during confirmed +playback or _1_ all the other times + +Known issues + +Mount and --dir-cache-time + +--dir-cache-time controls the first layer of directory caching which +works at the mount layer. Being an independent caching mechanism from +the cache backend, it will manage its own entries based on the +configured time. + +To avoid getting in a scenario where dir cache has obsolete data and +cache would have the correct one, try to set --dir-cache-time to a lower +time than --cache-info-age. Default values are already configured in +this way. + +Windows support - Experimental + +There are a couple of issues with Windows mount functionality that still +require some investigations. It should be considered as experimental +thus far as fixes come in for this OS. + +Most of the issues seem to be related to the difference between +filesystems on Linux flavors and Windows as cache is heavily dependant +on them. + +Any reports or feedback on how cache behaves on this OS is greatly +appreciated. + +- https://github.com/ncw/rclone/issues/1935 +- https://github.com/ncw/rclone/issues/1907 +- https://github.com/ncw/rclone/issues/1834 + +Risk of throttling + +Future iterations of the cache backend will make use of the pooling +functionality of the cloud provider to synchronize and at the same time +make writing through it more tolerant to failures. + +There are a couple of enhancements in track to add these but in the +meantime there is a valid concern that the expiring cache listings can +lead to cloud provider throttles or bans due to repeated queries on it +for very large mounts. + +Some recommendations: - don't use a very small interval for entry +informations (--cache-info-age) - while writes aren't yet optimised, you +can still write through cache which gives you the advantage of adding +the file in the cache at the same time if configured to do so. + +Future enhancements: + +- https://github.com/ncw/rclone/issues/1937 +- https://github.com/ncw/rclone/issues/1936 + +cache and crypt + +One common scenario is to keep your data encrypted in the cloud provider +using the crypt remote. crypt uses a similar technique to wrap around an +existing remote and handles this translation in a seamless way. + +There is an issue with wrapping the remotes in this order: CLOUD REMOTE +-> CRYPT -> CACHE + +During testing, I experienced a lot of bans with the remotes in this +order. I suspect it might be related to how crypt opens files on the +cloud provider which makes it think we're downloading the full file +instead of small chunks. Organizing the remotes in this order yelds +better results: CLOUD REMOTE -> CACHE -> CRYPT + +Cache and Remote Control (--rc) + +Cache supports the new --rc mode in rclone and can be remote controlled +through the following end points: By default, the listener is disabled +if you do not add the flag. + +rc cache/expire + +Purge a remote from the cache backend. Supports either a directory or a +file. It supports both encrypted and unencrypted file names if cache is +wrapped by crypt. + +Params: - REMOTE = path to remote (REQUIRED) - WITHDATA = true/false to +delete cached data (chunks) as well _(optional, false by default)_ + +Specific options + +Here are the command line options specific to this cloud storage system. + +--cache-db-path=PATH + +Path to where the file structure metadata (DB) is stored locally. The +remote name is used as the DB file name. + +DEFAULT: /cache-backend/ EXAMPLE: /.cache/cache-backend/test-cache + +--cache-chunk-path=PATH + +Path to where partial file data (chunks) is stored locally. The remote +name is appended to the final path. + +This config follows the --cache-db-path. If you specify a custom +location for --cache-db-path and don't specify one for +--cache-chunk-path then --cache-chunk-path will use the same path as +--cache-db-path. + +DEFAULT: /cache-backend/ EXAMPLE: /.cache/cache-backend/test-cache + +--cache-db-purge + +Flag to clear all the cached data for this remote before. + +DEFAULT: not set + +--cache-chunk-size=SIZE + +The size of a chunk (partial file data). Use lower numbers for slower +connections. If the chunk size is changed, any downloaded chunks will be +invalid and cache-chunk-path will need to be cleared or unexpected EOF +errors will occur. + +DEFAULT: 5M + +--cache-total-chunk-size=SIZE + +The total size that the chunks can take up on the local disk. If cache +exceeds this value then it will start to the delete the oldest chunks +until it goes under this value. + +DEFAULT: 10G + +--cache-chunk-clean-interval=DURATION + +How often should cache perform cleanups of the chunk storage. The +default value should be ok for most people. If you find that cache goes +over cache-total-chunk-size too often then try to lower this value to +force it to perform cleanups more often. + +DEFAULT: 1m + +--cache-info-age=DURATION + +How long to keep file structure information (directory listings, file +size, mod times etc) locally. + +If all write operations are done through cache then you can safely make +this value very large as the cache store will also be updated in real +time. + +DEFAULT: 6h + +--cache-read-retries=RETRIES + +How many times to retry a read from a cache storage. + +Since reading from a cache stream is independent from downloading file +data, readers can get to a point where there's no more data in the +cache. Most of the times this can indicate a connectivity issue if cache +isn't able to provide file data anymore. + +For really slow connections, increase this to a point where the stream +is able to provide data but your experience will be very stuttering. + +DEFAULT: 10 + +--cache-workers=WORKERS + +How many workers should run in parallel to download chunks. + +Higher values will mean more parallel processing (better CPU needed) and +more concurrent requests on the cloud provider. This impacts several +aspects like the cloud provider API limits, more stress on the hardware +that rclone runs on but it also means that streams will be more fluid +and data will be available much more faster to readers. + +NOTE: If the optional Plex integration is enabled then this setting will +adapt to the type of reading performed and the value specified here will +be used as a maximum number of workers to use. DEFAULT: 4 + +--cache-chunk-no-memory + +By default, cache will keep file data during streaming in RAM as well to +provide it to readers as fast as possible. + +This transient data is evicted as soon as it is read and the number of +chunks stored doesn't exceed the number of workers. However, depending +on other settings like cache-chunk-size and cache-workers this footprint +can increase if there are parallel streams too (multiple files being +read at the same time). + +If the hardware permits it, use this feature to provide an overall +better performance during streaming but it can also be disabled if RAM +is not available on the local machine. + +DEFAULT: not set + +--cache-rps=NUMBER + +This setting places a hard limit on the number of requests per second +that cache will be doing to the cloud provider remote and try to respect +that value by setting waits between reads. + +If you find that you're getting banned or limited on the cloud provider +through cache and know that a smaller number of requests per second will +allow you to work with it then you can use this setting for that. + +A good balance of all the other settings should make this setting +useless but it is available to set for more special cases. + +NOTE: This will limit the number of requests during streams but other +API calls to the cloud provider like directory listings will still pass. + +DEFAULT: disabled + +--cache-writes + +If you need to read files immediately after you upload them through +cache you can enable this flag to have their data stored in the cache +store at the same time during upload. + +DEFAULT: not set + +--cache-tmp-upload-path=PATH + +This is the path where cache will use as a temporary storage for new +files that need to be uploaded to the cloud provider. + +Specifying a value will enable this feature. Without it, it is +completely disabled and files will be uploaded directly to the cloud +provider + +DEFAULT: empty + +--cache-tmp-wait-time=DURATION + +This is the duration that a file must wait in the temporary location +_cache-tmp-upload-path_ before it is selected for upload. + +Note that only one file is uploaded at a time and it can take longer to +start the upload if a queue formed for this purpose. + +DEFAULT: 15m + +--cache-db-wait-time=DURATION + +Only one process can have the DB open at any one time, so rclone waits +for this duration for the DB to become available before it gives an +error. + +If you set it to 0 then it will wait forever. + +DEFAULT: 1s + + +Crypt + +The crypt remote encrypts and decrypts another remote. + +To use it first set up the underlying remote following the config +instructions for that remote. You can also use a local pathname instead +of a remote which will encrypt and decrypt from that directory which +might be useful for encrypting onto a USB stick for example. + +First check your chosen remote is working - we'll call it remote:path in +these docs. Note that anything inside remote:path will be encrypted and +anything outside won't. This means that if you are using a bucket based +remote (eg S3, B2, swift) then you should probably put the bucket in the +remote s3:bucket. If you just use s3: then rclone will make encrypted +bucket names too (if using file name encryption) which may or may not be +what you want. + +Now configure crypt using rclone config. We will call this one secret to +differentiate it from the remote. + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> secret + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" + 10 / Microsoft OneDrive + \ "onedrive" + 11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 12 / SSH/SFTP Connection + \ "sftp" + 13 / Yandex Disk + \ "yandex" + Storage> 5 + Remote to encrypt/decrypt. + Normally should contain a ':' and a path, eg "myremote:path/to/dir", + "myremote:bucket" or maybe "myremote:" (not recommended). + remote> remote:path + How to encrypt the filenames. + Choose a number from below, or type in your own value + 1 / Don't encrypt the file names. Adds a ".bin" extension only. + \ "off" + 2 / Encrypt the filenames see the docs for the details. + \ "standard" + 3 / Very simple filename obfuscation. + \ "obfuscate" + filename_encryption> 2 + Option to either encrypt directory names or leave them intact. + Choose a number from below, or type in your own value + 1 / Encrypt directory names. + \ "true" + 2 / Don't encrypt directory names, leave them intact. + \ "false" + filename_encryption> 1 + Password or pass phrase for encryption. + y) Yes type in my own password + g) Generate random password + y/g> y + Enter the password: + password: + Confirm the password: + password: + Password or pass phrase for salt. Optional but recommended. + Should be different to the previous password. + y) Yes type in my own password + g) Generate random password + n) No leave this optional password blank + y/g/n> g + Password strength in bits. + 64 is just about memorable + 128 is secure + 1024 is the maximum + Bits> 128 + Your password is: JAsJvRcgR-_veXNfy_sGmQ + Use this password? + y) Yes + n) No + y/n> y + Remote config + -------------------- + [secret] + remote = remote:path + filename_encryption = standard + password = *** ENCRYPTED *** + password2 = *** ENCRYPTED *** + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +IMPORTANT The password is stored in the config file is lightly obscured +so it isn't immediately obvious what it is. It is in no way secure +unless you use config file encryption. + +A long passphrase is recommended, or you can use a random one. Note that +if you reconfigure rclone with the same passwords/passphrases elsewhere +it will be compatible - all the secrets used are derived from those two +passwords/passphrases. + +Note that rclone does not encrypt + +- file length - this can be calcuated within 16 bytes +- modification time - used for syncing + + +Specifying the remote + +In normal use, make sure the remote has a : in. If you specify the +remote without a : then rclone will use a local directory of that name. +So if you use a remote of /path/to/secret/files then rclone will encrypt +stuff to that directory. If you use a remote of name then rclone will +put files in a directory called name in the current directory. + +If you specify the remote as remote:path/to/dir then rclone will store +encrypted files in path/to/dir on the remote. If you are using file name +encryption, then when you save files to secret:subdir/subfile this will +store them in the unencrypted path path/to/dir but the subdir/subpath +bit will be encrypted. + +Note that unless you want encrypted bucket names (which are difficult to +manage because you won't know what directory they represent in web +interfaces etc), you should probably specify a bucket, eg +remote:secretbucket when using bucket based remotes such as S3, Swift, +Hubic, B2, GCS. + + +Example + +To test I made a little directory of files using "standard" file name +encryption. + + plaintext/ + ├── file0.txt + ├── file1.txt + └── subdir + ├── file2.txt + ├── file3.txt + └── subsubdir + └── file4.txt + +Copy these to the remote and list them back + + $ rclone -q copy plaintext secret: + $ rclone -q ls secret: + 7 file1.txt + 6 file0.txt + 8 subdir/file2.txt + 10 subdir/subsubdir/file4.txt + 9 subdir/file3.txt + +Now see what that looked like when encrypted + + $ rclone -q ls remote:path + 55 hagjclgavj2mbiqm6u6cnjjqcg + 54 v05749mltvv1tf4onltun46gls + 57 86vhrsv86mpbtd3a0akjuqslj8/dlj7fkq4kdq72emafg7a7s41uo + 58 86vhrsv86mpbtd3a0akjuqslj8/7uu829995du6o42n32otfhjqp4/b9pausrfansjth5ob3jkdqd4lc + 56 86vhrsv86mpbtd3a0akjuqslj8/8njh1sk437gttmep3p70g81aps + +Note that this retains the directory structure which means you can do +this + + $ rclone -q ls secret:subdir + 8 file2.txt + 9 file3.txt + 10 subsubdir/file4.txt + +If don't use file name encryption then the remote will look like this - +note the .bin extensions added to prevent the cloud provider attempting +to interpret the data. + + $ rclone -q ls remote:path + 54 file0.txt.bin + 57 subdir/file3.txt.bin + 56 subdir/file2.txt.bin + 58 subdir/subsubdir/file4.txt.bin + 55 file1.txt.bin + +File name encryption modes + +Here are some of the features of the file name encryption modes + +Off + +- doesn't hide file names or directory structure +- allows for longer file names (~246 characters) +- can use sub paths and copy single files + +Standard + +- file names encrypted +- file names can't be as long (~143 characters) +- can use sub paths and copy single files +- directory structure visible +- identical files names will have identical uploaded names +- can use shortcuts to shorten the directory recursion + +Obfuscation + +This is a simple "rotate" of the filename, with each file having a rot +distance based on the filename. We store the distance at the beginning +of the filename. So a file called "hello" may become "53.jgnnq" + +This is not a strong encryption of filenames, but it may stop automated +scanning tools from picking up on filename patterns. As such it's an +intermediate between "off" and "standard". The advantage is that it +allows for longer path segment names. + +There is a possibility with some unicode based filenames that the +obfuscation is weak and may map lower case characters to upper case +equivalents. You can not rely on this for strong protection. + +- file names very lightly obfuscated +- file names can be longer than standard encryption +- can use sub paths and copy single files +- directory structure visible +- identical files names will have identical uploaded names + +Cloud storage systems have various limits on file name length and total +path length which you are more likely to hit using "Standard" file name +encryption. If you keep your file names to below 156 characters in +length then you should be OK on all providers. + +There may be an even more secure file name encryption mode in the future +which will address the long file name problem. + +Directory name encryption + +Crypt offers the option of encrypting dir names or leaving them intact. +There are two options: + +True + +Encrypts the whole file path including directory names Example: +1/12/123.txt is encrypted to +p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0 + +False + +Only encrypts file names, skips directory names Example: 1/12/123.txt is +encrypted to 1/12/qgm4avr35m5loi1th53ato71v0 + +Modified time and hashes + +Crypt stores modification times using the underlying remote so support +depends on that. + +Hashes are not stored for crypt. However the data integrity is protected +by an extremely strong crypto authenticator. + +Note that you should use the rclone cryptcheck command to check the +integrity of a crypted remote instead of rclone check which can't check +the checksums properly. + +Specific options + +Here are the command line options specific to this cloud storage system. + +--crypt-show-mapping + +If this flag is set then for each file that the remote is asked to list, +it will log (at level INFO) a line stating the decrypted file name and +the encrypted file name. + +This is so you can work out which encrypted names are which decrypted +names just in case you need to do something with the encrypted file +names, or for debugging purposes. + + +Backing up a crypted remote + +If you wish to backup a crypted remote, it it recommended that you use +rclone sync on the encrypted files, and make sure the passwords are the +same in the new encrypted remote. + +This will have the following advantages + +- rclone sync will check the checksums while copying +- you can use rclone check between the encrypted remotes +- you don't decrypt and encrypt unnecessarily + +For example, let's say you have your original remote at remote: with the +encrypted version at eremote: with path remote:crypt. You would then set +up the new remote remote2: and then the encrypted version eremote2: with +path remote2:crypt using the same passwords as eremote:. + +To sync the two remotes you would do + + rclone sync remote:crypt remote2:crypt + +And to check the integrity you would do + + rclone check remote:crypt remote2:crypt + + +File formats + +File encryption + +Files are encrypted 1:1 source file to destination object. The file has +a header and is divided into chunks. + +Header + +- 8 bytes magic string RCLONE\x00\x00 +- 24 bytes Nonce (IV) + +The initial nonce is generated from the operating systems crypto strong +random number generator. The nonce is incremented for each chunk read +making sure each nonce is unique for each block written. The chance of a +nonce being re-used is minuscule. If you wrote an exabyte of data (10¹⁸ +bytes) you would have a probability of approximately 2×10⁻³² of re-using +a nonce. + +Chunk + +Each chunk will contain 64kB of data, except for the last one which may +have less data. The data chunk is in standard NACL secretbox format. +Secretbox uses XSalsa20 and Poly1305 to encrypt and authenticate +messages. + +Each chunk contains: + +- 16 Bytes of Poly1305 authenticator +- 1 - 65536 bytes XSalsa20 encrypted data + +64k chunk size was chosen as the best performing chunk size (the +authenticator takes too much time below this and the performance drops +off due to cache effects above this). Note that these chunks are +buffered in memory so they can't be too big. + +This uses a 32 byte (256 bit key) key derived from the user password. + +Examples + +1 byte file will encrypt to + +- 32 bytes header +- 17 bytes data chunk + +49 bytes total + +1MB (1048576 bytes) file will encrypt to + +- 32 bytes header +- 16 chunks of 65568 bytes + +1049120 bytes total (a 0.05% overhead). This is the overhead for big +files. + +Name encryption + +File names are encrypted segment by segment - the path is broken up into +/ separated strings and these are encrypted individually. + +File segments are padded using using PKCS#7 to a multiple of 16 bytes +before encryption. + +They are then encrypted with EME using AES with 256 bit key. EME +(ECB-Mix-ECB) is a wide-block encryption mode presented in the 2003 +paper "A Parallelizable Enciphering Mode" by Halevi and Rogaway. + +This makes for deterministic encryption which is what we want - the same +filename must encrypt to the same thing otherwise we can't find it on +the cloud storage system. + +This means that + +- filenames with the same name will encrypt the same +- filenames which start the same won't have a common prefix + +This uses a 32 byte key (256 bits) and a 16 byte (128 bits) IV both of +which are derived from the user password. + +After encryption they are written out using a modified version of +standard base32 encoding as described in RFC4648. The standard encoding +is modified in two ways: + +- it becomes lower case (no-one likes upper case filenames!) +- we strip the padding character = + +base32 is used rather than the more efficient base64 so rclone can be +used on case insensitive remotes (eg Windows, Amazon Drive). + +Key derivation + +Rclone uses scrypt with parameters N=16384, r=8, p=1 with an optional +user supplied salt (password2) to derive the 32+32+16 = 80 bytes of key +material required. If the user doesn't supply a salt then rclone uses an +internal one. + +scrypt makes it impractical to mount a dictionary attack on rclone +encrypted data. For full protection against this you should always use a +salt. + + +Dropbox + +Paths are specified as remote:path + +Dropbox paths may be as deep as required, eg +remote:directory/subdirectory. + +The initial setup for dropbox involves getting a token from Dropbox +which you need to do in your browser. rclone config walks you through +it. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + n) New remote + d) Delete remote + q) Quit config + e/n/d/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" + 10 / Microsoft OneDrive + \ "onedrive" + 11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 12 / SSH/SFTP Connection + \ "sftp" + 13 / Yandex Disk + \ "yandex" + Storage> 4 + Dropbox App Key - leave blank normally. + app_key> + Dropbox App Secret - leave blank normally. + app_secret> + Remote config + Please visit: + https://www.dropbox.com/1/oauth2/authorize?client_id=XXXXXXXXXXXXXXX&response_type=code + Enter the code: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXXXXXXXX + -------------------- + [remote] + app_key = + app_secret = + token = XXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXX_XXXXXXXXXXXXXXXXXXXXXXXXXXXXX + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +You can then use it like this, + +List directories in top level of your dropbox + + rclone lsd remote: + +List all the files in your dropbox + + rclone ls remote: + +To copy a local directory to a dropbox directory called backup + + rclone copy /home/source remote:backup + +Dropbox for business + +Rclone supports Dropbox for business and Team Folders. + +When using Dropbox for business remote: and remote:path/to/file will +refer to your personal folder. + +If you wish to see Team Folders you must use a leading / in the path, so +rclone lsd remote:/ will refer to the root and show you all Team Folders +and your User Folder. + +You can then use team folders like this remote:/TeamFolder and +remote:/TeamFolder/path/to/file. + +A leading / for a Dropbox personal account will do nothing, but it will +take an extra HTTP transaction so it should be avoided. + +Modified time and Hashes + +Dropbox supports modified times, but the only way to set a modification +time is to re-upload the file. + +This means that if you uploaded your data with an older version of +rclone which didn't support the v2 API and modified times, rclone will +decide to upload all your old data to fix the modification times. If you +don't want this to happen use --size-only or --checksum flag to stop it. + +Dropbox supports its own hash type which is checked for all transfers. + +Specific options + +Here are the command line options specific to this cloud storage system. + +--dropbox-chunk-size=SIZE + +Any files larger than this will be uploaded in chunks of this size. The +default is 48MB. The maximum is 150MB. + +Note that chunks are buffered in memory (one at a time) so rclone can +deal with retries. Setting this larger will increase the speed slightly +(at most 10% for 128MB in tests) at the cost of using more memory. It +can be set smaller if you are tight on memory. + +Limitations + +Note that Dropbox is case insensitive so you can't have a file called +"Hello.doc" and one called "hello.doc". + +There are some file names such as thumbs.db which Dropbox can't store. +There is a full list of them in the "Ignored Files" section of this +document. Rclone will issue an error message +File name disallowed - not uploading if it attempts to upload one of +those file names, but the sync won't fail. + +If you have more than 10,000 files in a directory then +rclone purge dropbox:dir will return the error +Failed to purge: There are too many files involved in this operation. As +a work-around do an rclone delete dropbox:dir followed by an +rclone rmdir dropbox:dir. + + +FTP + +FTP is the File Transfer Protocol. FTP support is provided using the +github.com/jlaffaye/ftp package. + +Here is an example of making an FTP configuration. First run + + rclone config + +This will guide you through an interactive setup process. An FTP remote +only needs a host together with and a username and a password. With +anonymous FTP server, you will need to use anonymous as username and +your email address as the password. + + No remotes found - make a new one + n) New remote + r) Rename remote + c) Copy remote + s) Set configuration password + q) Quit config + n/r/c/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / FTP Connection + \ "ftp" + 7 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 8 / Google Drive + \ "drive" + 9 / Hubic + \ "hubic" + 10 / Local Disk + \ "local" + 11 / Microsoft OneDrive + \ "onedrive" + 12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 13 / SSH/SFTP Connection + \ "sftp" + 14 / Yandex Disk + \ "yandex" + Storage> ftp + FTP host to connect to + Choose a number from below, or type in your own value + 1 / Connect to ftp.example.com + \ "ftp.example.com" + host> ftp.example.com + FTP username, leave blank for current username, ncw + user> + FTP port, leave blank to use default (21) + port> + FTP password + y) Yes type in my own password + g) Generate random password + y/g> y + Enter the password: + password: + Confirm the password: + password: + Remote config + -------------------- + [remote] + host = ftp.example.com + user = + port = + pass = *** ENCRYPTED *** + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +This remote is called remote and can now be used like this + +See all directories in the home directory + + rclone lsd remote: + +Make a new directory + + rclone mkdir remote:path/to/directory + +List the contents of a directory + + rclone ls remote:path/to/directory + +Sync /home/local/directory to the remote directory, deleting any excess +files in the directory. + + rclone sync /home/local/directory remote:directory + +Modified time + +FTP does not support modified times. Any times you see on the server +will be time of upload. + +Checksums + +FTP does not support any checksums. + +Limitations + +Note that since FTP isn't HTTP based the following flags don't work with +it: --dump-headers, --dump-bodies, --dump-auth + +Note that --timeout isn't supported (but --contimeout is). + +Note that --bind isn't supported. + +FTP could support server side move but doesn't yet. + + +Google Cloud Storage + +Paths are specified as remote:bucket (or remote: for the lsd command.) +You may put subdirectories in too, eg remote:bucket/path/to/dir. + +The initial setup for google cloud storage involves getting a token from +Google Cloud Storage which you need to do in your browser. rclone config +walks you through it. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + n) New remote + d) Delete remote + q) Quit config + e/n/d/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" + 10 / Microsoft OneDrive + \ "onedrive" + 11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 12 / SSH/SFTP Connection + \ "sftp" + 13 / Yandex Disk + \ "yandex" + Storage> 6 + Google Application Client Id - leave blank normally. + client_id> + Google Application Client Secret - leave blank normally. + client_secret> + Project number optional - needed only for list/create/delete buckets - see your developer console. + project_number> 12345678 + Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. + service_account_file> + Access Control List for new objects. + Choose a number from below, or type in your own value + 1 / Object owner gets OWNER access, and all Authenticated Users get READER access. + \ "authenticatedRead" + 2 / Object owner gets OWNER access, and project team owners get OWNER access. + \ "bucketOwnerFullControl" + 3 / Object owner gets OWNER access, and project team owners get READER access. + \ "bucketOwnerRead" + 4 / Object owner gets OWNER access [default if left blank]. + \ "private" + 5 / Object owner gets OWNER access, and project team members get access according to their roles. + \ "projectPrivate" + 6 / Object owner gets OWNER access, and all Users get READER access. + \ "publicRead" + object_acl> 4 + Access Control List for new buckets. + Choose a number from below, or type in your own value + 1 / Project team owners get OWNER access, and all Authenticated Users get READER access. + \ "authenticatedRead" + 2 / Project team owners get OWNER access [default if left blank]. + \ "private" + 3 / Project team members get access according to their roles. + \ "projectPrivate" + 4 / Project team owners get OWNER access, and all Users get READER access. + \ "publicRead" + 5 / Project team owners get OWNER access, and all Users get WRITER access. + \ "publicReadWrite" + bucket_acl> 2 + Location for the newly created buckets. + Choose a number from below, or type in your own value + 1 / Empty for default location (US). + \ "" + 2 / Multi-regional location for Asia. + \ "asia" + 3 / Multi-regional location for Europe. + \ "eu" + 4 / Multi-regional location for United States. + \ "us" + 5 / Taiwan. + \ "asia-east1" + 6 / Tokyo. + \ "asia-northeast1" + 7 / Singapore. + \ "asia-southeast1" + 8 / Sydney. + \ "australia-southeast1" + 9 / Belgium. + \ "europe-west1" + 10 / London. + \ "europe-west2" + 11 / Iowa. + \ "us-central1" + 12 / South Carolina. + \ "us-east1" + 13 / Northern Virginia. + \ "us-east4" + 14 / Oregon. + \ "us-west1" + location> 12 + The storage class to use when storing objects in Google Cloud Storage. + Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Multi-regional storage class + \ "MULTI_REGIONAL" + 3 / Regional storage class + \ "REGIONAL" + 4 / Nearline storage class + \ "NEARLINE" + 5 / Coldline storage class + \ "COLDLINE" + 6 / Durable reduced availability storage class + \ "DURABLE_REDUCED_AVAILABILITY" + storage_class> 5 + Remote config + Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine or Y didn't work + y) Yes + n) No + y/n> y + If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth + Log in and authorize rclone for access + Waiting for code... + Got code + -------------------- + [remote] + type = google cloud storage + client_id = + client_secret = + token = {"AccessToken":"xxxx.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","RefreshToken":"x/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_xxxxxxxxx","Expiry":"2014-07-17T20:49:14.929208288+01:00","Extra":null} + project_number = 12345678 + object_acl = private + bucket_acl = private + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Google if you use auto config mode. This only +runs from the moment it opens your browser to the moment you get back +the verification code. This is on http://127.0.0.1:53682/ and this it +may require you to unblock it temporarily if you are running a host +firewall, or use manual mode. + +This remote is called remote and can now be used like this + +See all the buckets in your project + + rclone lsd remote: + +Make a new bucket + + rclone mkdir remote:bucket + +List the contents of a bucket + + rclone ls remote:bucket + +Sync /home/local/directory to the remote bucket, deleting any excess +files in the bucket. + + rclone sync /home/local/directory remote:bucket + +Service Account support + +You can set up rclone with Google Cloud Storage in an unattended mode, +i.e. not tied to a specific end-user Google account. This is useful when +you want to synchronise files onto machines that don't have actively +logged-in users, for example build machines. + +To get credentials for Google Cloud Platform IAM Service Accounts, +please head to the Service Account section of the Google Developer +Console. Service Accounts behave just like normal User permissions in +Google Cloud Storage ACLs, so you can limit their access (e.g. make them +read only). After creating an account, a JSON file containing the +Service Account's credentials will be downloaded onto your machines. +These credentials are what rclone will use for authentication. + +To use a Service Account instead of OAuth2 token flow, enter the path to +your Service Account credentials at the service_account_file prompt and +rclone won't use the browser based authentication flow. If you'd rather +stuff the contents of the credentials file into the rclone config file, +you can set service_account_credentials with the actual contents of the +file instead, or set the equivalent environment variable. + +--fast-list + +This remote supports --fast-list which allows you to use fewer +transactions in exchange for more memory. See the rclone docs for more +details. + +Modified time + +Google google cloud storage stores md5sums natively and rclone stores +modification times as metadata on the object, under the "mtime" key in +RFC3339 format accurate to 1ns. + + +Google Drive + +Paths are specified as drive:path + +Drive paths may be as deep as required, eg drive:directory/subdirectory. + +The initial setup for drive involves getting a token from Google drive +which you need to do in your browser. rclone config walks you through +it. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + r) Rename remote + c) Copy remote + s) Set configuration password + q) Quit config + n/r/c/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + [snip] + 10 / Google Drive + \ "drive" + [snip] + Storage> drive + Google Application Client Id - leave blank normally. + client_id> + Google Application Client Secret - leave blank normally. + client_secret> + Scope that rclone should use when requesting access from drive. + Choose a number from below, or type in your own value + 1 / Full access all files, excluding Application Data Folder. + \ "drive" + 2 / Read-only access to file metadata and file contents. + \ "drive.readonly" + / Access to files created by rclone only. + 3 | These are visible in the drive website. + | File authorization is revoked when the user deauthorizes the app. + \ "drive.file" + / Allows read and write access to the Application Data folder. + 4 | This is not visible in the drive website. + \ "drive.appfolder" + / Allows read-only access to file metadata but + 5 | does not allow any access to read or download file content. + \ "drive.metadata.readonly" + scope> 1 + ID of the root folder - leave blank normally. Fill in to access "Computers" folders. (see docs). + root_folder_id> + Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. + service_account_file> + Remote config + Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine or Y didn't work + y) Yes + n) No + y/n> y + If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth + Log in and authorize rclone for access + Waiting for code... + Got code + Configure this as a team drive? + y) Yes + n) No + y/n> n + -------------------- + [remote] + client_id = + client_secret = + scope = drive + root_folder_id = + service_account_file = + token = {"access_token":"XXX","token_type":"Bearer","refresh_token":"XXX","expiry":"2014-03-16T13:57:58.955387075Z"} + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Google if you use auto config mode. This only +runs from the moment it opens your browser to the moment you get back +the verification code. This is on http://127.0.0.1:53682/ and this it +may require you to unblock it temporarily if you are running a host +firewall, or use manual mode. + +You can then use it like this, + +List directories in top level of your drive + + rclone lsd remote: + +List all the files in your drive + + rclone ls remote: + +To copy a local directory to a drive directory called backup + + rclone copy /home/source remote:backup + +Scopes + +Rclone allows you to select which scope you would like for rclone to +use. This changes what type of token is granted to rclone. The scopes +are defined here.. + +The scope are + +drive + +This is the default scope and allows full access to all files, except +for the Application Data Folder (see below). + +Choose this one if you aren't sure. + +drive.readonly + +This allows read only access to all files. Files may be listed and +downloaded but not uploaded, renamed or deleted. + +drive.file + +With this scope rclone can read/view/modify only those files and folders +it creates. + +So if you uploaded files to drive via the web interface (or any other +means) they will not be visible to rclone. + +This can be useful if you are using rclone to backup data and you want +to be sure confidential data on your drive is not visible to rclone. + +Files created with this scope are visible in the web interface. + +drive.appfolder + +This gives rclone its own private area to store files. Rclone will not +be able to see any other files on your drive and you won't be able to +see rclone's files from the web interface either. + +drive.metadata.readonly + +This allows read only access to file names only. It does not allow +rclone to download or upload data, or rename or delete files or +directories. + +Root folder ID + +You can set the root_folder_id for rclone. This is the directory +(identified by its Folder ID) that rclone considers to be a the root of +your drive. + +Normally you will leave this blank and rclone will determine the correct +root to use itself. + +However you can set this to restrict rclone to a specific folder +hierarchy or to access data within the "Computers" tab on the drive web +interface (where files from Google's Backup and Sync desktop program +go). + +In order to do this you will have to find the Folder ID of the directory +you wish rclone to display. This will be the last segment of the URL +when you open the relevant folder in the drive web interface. + +So if the folder you want rclone to use has a URL which looks like +https://drive.google.com/drive/folders/1XyfxxxxxxxxxxxxxxxxxxxxxxxxxKHCh +in the browser, then you use 1XyfxxxxxxxxxxxxxxxxxxxxxxxxxKHCh as the +root_folder_id in the config. + +NB folders under the "Computers" tab seem to be read only (drive gives a +500 error) when using rclone. + +There doesn't appear to be an API to discover the folder IDs of the +"Computers" tab - please contact us if you know otherwise! + +Note also that rclone can't access any data under the "Backups" tab on +the google drive web interface yet. + +Service Account support + +You can set up rclone with Google Drive in an unattended mode, i.e. not +tied to a specific end-user Google account. This is useful when you want +to synchronise files onto machines that don't have actively logged-in +users, for example build machines. + +To use a Service Account instead of OAuth2 token flow, enter the path to +your Service Account credentials at the service_account_file prompt +during rclone config and rclone won't use the browser based +authentication flow. If you'd rather stuff the contents of the +credentials file into the rclone config file, you can set +service_account_credentials with the actual contents of the file +instead, or set the equivalent environment variable. + +Use case - Google Apps/G-suite account and individual Drive + +Let's say that you are the administrator of a Google Apps (old) or +G-suite account. The goal is to store data on an individual's Drive +account, who IS a member of the domain. We'll call the domain +EXAMPLE.COM, and the user FOO@EXAMPLE.COM. + +There's a few steps we need to go through to accomplish this: + +1. Create a service account for example.com + +- To create a service account and obtain its credentials, go to the + Google Developer Console. +- You must have a project - create one if you don't. +- Then go to "IAM & admin" -> "Service Accounts". +- Use the "Create Credentials" button. Fill in "Service account name" + with something that identifies your client. "Role" can be empty. +- Tick "Furnish a new private key" - select "Key type JSON". +- Tick "Enable G Suite Domain-wide Delegation". This option makes + "impersonation" possible, as documented here: Delegating domain-wide + authority to the service account +- These credentials are what rclone will use for authentication. If + you ever need to remove access, press the "Delete service account + key" button. + +2. Allowing API access to example.com Google Drive + +- Go to example.com's admin console +- Go into "Security" (or use the search bar) +- Select "Show more" and then "Advanced settings" +- Select "Manage API client access" in the "Authentication" section +- In the "Client Name" field enter the service account's "Client ID" - + this can be found in the Developer Console under "IAM & Admin" -> + "Service Accounts", then "View Client ID" for the newly created + service account. It is a ~21 character numerical string. +- In the next field, "One or More API Scopes", enter + https://www.googleapis.com/auth/drive to grant access to Google + Drive specifically. + +3. Configure rclone, assuming a new install + + rclone config + + n/s/q> n # New + name>gdrive # Gdrive is an example name + Storage> # Select the number shown for Google Drive + client_id> # Can be left blank + client_secret> # Can be left blank + scope> # Select your scope, 1 for example + root_folder_id> # Can be left blank + service_account_file> /home/foo/myJSONfile.json # This is where the JSON file goes! + y/n> # Auto config, y + +4. Verify that it's working + +- rclone -v --drive-impersonate foo@example.com lsf gdrive:backup +- The arguments do: + - -v - verbose logging + - --drive-impersonate foo@example.com - this is what does the + magic, pretending to be user foo. + - lsf - list files in a parsing friendly way + - gdrive:backup - use the remote called gdrive, work in the folder + named backup. + +Team drives + +If you want to configure the remote to point to a Google Team Drive then +answer y to the question Configure this as a team drive?. + +This will fetch the list of Team Drives from google and allow you to +configure which one you want to use. You can also type in a team drive +ID if you prefer. + +For example: + + Configure this as a team drive? + y) Yes + n) No + y/n> y + Fetching team drive list... + Choose a number from below, or type in your own value + 1 / Rclone Test + \ "xxxxxxxxxxxxxxxxxxxx" + 2 / Rclone Test 2 + \ "yyyyyyyyyyyyyyyyyyyy" + 3 / Rclone Test 3 + \ "zzzzzzzzzzzzzzzzzzzz" + Enter a Team Drive ID> 1 + -------------------- + [remote] + client_id = + client_secret = + token = {"AccessToken":"xxxx.x.xxxxx_xxxxxxxxxxx_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","RefreshToken":"1/xxxxxxxxxxxxxxxx_xxxxxxxxxxxxxxxxxxxxxxxxxx","Expiry":"2014-03-16T13:57:58.955387075Z","Extra":null} + team_drive = xxxxxxxxxxxxxxxxxxxx + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +--fast-list + +This remote supports --fast-list which allows you to use fewer +transactions in exchange for more memory. See the rclone docs for more +details. + +It does this by combining multiple list calls into a single API request. + +This works by combining many '%s' in parents filters into one +expression. To list the contents of directories a, b and c, the the +following requests will be send by the regular List function: + + trashed=false and 'a' in parents + trashed=false and 'b' in parents + trashed=false and 'c' in parents + +These can now be combined into a single request: + + trashed=false and ('a' in parents or 'b' in parents or 'c' in parents) + +The implementation of ListR will put up to 50 parents filters into one +request. It will use the --checkers value to specify the number of +requests to run in parallel. + +In tests, these batch requests were up to 20x faster than the regular +method. Running the following command against different sized folders +gives: + + rclone lsjson -vv -R --checkers=6 gdrive:folder + +small folder (220 directories, 700 files): + +- without --fast-list: 38s +- with --fast-list: 10s + +large folder (10600 directories, 39000 files): + +- without --fast-list: 22:05 min +- with --fast-list: 58s + +Modified time + +Google drive stores modification times accurate to 1 ms. + +Revisions + +Google drive stores revisions of files. When you upload a change to an +existing file to google drive using rclone it will create a new revision +of that file. + +Revisions follow the standard google policy which at time of writing was + +- They are deleted after 30 days or 100 revisions (whatever comes + first). +- They do not count towards a user storage quota. + +Deleting files + +By default rclone will send all files to the trash when deleting files. +If deleting them permanently is required then use the +--drive-use-trash=false flag, or set the equivalent environment +variable. + +Emptying trash + +If you wish to empty your trash you can use the rclone cleanup remote: +command which will permanently delete all your trashed files. This +command does not take any path arguments. + +Quota information + +To view your current quota you can use the rclone about remote: command +which will display your usage limit (quota), the usage in Google Drive, +the size of all files in the Trash and the space used by other Google +services such as Gmail. This command does not take any path arguments. + +Specific options + +Here are the command line options specific to this cloud storage system. + +--drive-acknowledge-abuse + +If downloading a file returns the error +This file has been identified as malware or spam and cannot be downloaded +with the error code cannotDownloadAbusiveFile then supply this flag to +rclone to indicate you acknowledge the risks of downloading the file and +rclone will download it anyway. + +--drive-auth-owner-only + +Only consider files owned by the authenticated user. + +--drive-chunk-size=SIZE + +Upload chunk size. Must a power of 2 >= 256k. Default value is 8 MB. + +Making this larger will improve performance, but note that each chunk is +buffered in memory one per transfer. + +Reducing this will reduce memory usage but decrease performance. + +--drive-formats + +Google documents can only be exported from Google drive. When rclone +downloads a Google doc it chooses a format to download depending upon +this setting. + +By default the formats are docx,xlsx,pptx,svg which are a sensible +default for an editable document. + +When choosing a format, rclone runs down the list provided in order and +chooses the first file format the doc can be exported as from the list. +If the file can't be exported to a format on the formats list, then +rclone will choose a format from the default list. + +If you prefer an archive copy then you might use --drive-formats pdf, or +if you prefer openoffice/libreoffice formats you might use +--drive-formats ods,odt,odp. + +Note that rclone adds the extension to the google doc, so if it is +calles My Spreadsheet on google docs, it will be exported as +My Spreadsheet.xlsx or My Spreadsheet.pdf etc. + +Here are the possible extensions with their corresponding mime types. + + Extension Mime Type Description + --------------------------------------------------------------------------- ------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------------- + csv text/csv Standard CSV format for Spreadsheets + doc application/msword Micosoft Office Document + docx application/vnd.openxmlformats-officedocument.wordprocessingml.document Microsoft Office Document + epub application/epub+zip E-book format + html text/html An HTML Document + jpg image/jpeg A JPEG Image File + odp application/vnd.oasis.opendocument.presentation Openoffice Presentation + ods application/vnd.oasis.opendocument.spreadsheet Openoffice Spreadsheet + ods application/x-vnd.oasis.opendocument.spreadsheet Openoffice Spreadsheet + odt application/vnd.oasis.opendocument.text Openoffice Document + pdf application/pdf Adobe PDF Format + png image/png PNG Image Format + pptx application/vnd.openxmlformats-officedocument.presentationml.presentation Microsoft Office Powerpoint + rtf application/rtf Rich Text Format + svg image/svg+xml Scalable Vector Graphics Format + tsv text/tab-separated-values Standard TSV format for spreadsheets + txt text/plain Plain Text + xls application/vnd.ms-excel Microsoft Office Spreadsheet + xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet Microsoft Office Spreadsheet + zip application/zip A ZIP file of HTML, Images CSS + +--drive-alternate-export + +If this option is set this instructs rclone to use an alternate set of +export URLs for drive documents. Users have reported that the official +export URLs can't export large documents, whereas these unofficial ones +can. + +See rclone issue #2243 for background, this google drive issue and this +helpful post. + +--drive-impersonate user + +When using a service account, this instructs rclone to impersonate the +user passed in. + +--drive-keep-revision-forever + +Keeps new head revision of the file forever. + +--drive-list-chunk int + +Size of listing chunk 100-1000. 0 to disable. (default 1000) + +--drive-shared-with-me + +Instructs rclone to operate on your "Shared with me" folder (where +Google Drive lets you access the files and folders others have shared +with you). + +This works both with the "list" (lsd, lsl, etc) and the "copy" commands +(copy, sync, etc), and with all other commands too. + +--drive-skip-gdocs + +Skip google documents in all listings. If given, gdocs practically +become invisible to rclone. + +--drive-trashed-only + +Only show files that are in the trash. This will show trashed files in +their original directory structure. + +--drive-upload-cutoff=SIZE + +File size cutoff for switching to chunked upload. Default is 8 MB. + +--drive-use-trash + +Controls whether files are sent to the trash or deleted permanently. +Defaults to true, namely sending files to the trash. Use +--drive-use-trash=false to delete files permanently instead. + +--drive-use-created-date + +Use the file creation date in place of the modification date. Defaults +to false. + +Useful when downloading data and you want the creation date used in +place of the last modified date. + +WARNING: This flag may have some unexpected consequences. + +When uploading to your drive all files will be overwritten unless they +haven't been modified since their creation. And the inverse will occur +while downloading. This side effect can be avoided by using the +--checksum flag. + +This feature was implemented to retain photos capture date as recorded +by google photos. You will first need to check the "Create a Google +Photos folder" option in your google drive settings. You can then copy +or move the photos locally and use the date the image was taken +(created) set as the modification date. + +Limitations + +Drive has quite a lot of rate limiting. This causes rclone to be limited +to transferring about 2 files per second only. Individual files may be +transferred much faster at 100s of MBytes/s but lots of small files can +take a long time. + +Server side copies are also subject to a separate rate limit. If you see +User rate limit exceeded errors, wait at least 24 hours and retry. You +can disable server side copies with --disable copy to download and +upload the files if you prefer. + +Limitations of Google Docs + +Google docs will appear as size -1 in rclone ls and as size 0 in +anything which uses the VFS layer, eg rclone mount, rclone serve. + +This is because rclone can't find out the size of the Google docs +without downloading them. + +Google docs will transfer correctly with rclone sync, rclone copy etc as +rclone knows to ignore the size when doing the transfer. + +However an unfortunate consequence of this is that you can't download +Google docs using rclone mount - you will get a 0 sized file. If you try +again the doc may gain its correct size and be downloadable. + +Duplicated files + +Sometimes, for no reason I've been able to track down, drive will +duplicate a file that rclone uploads. Drive unlike all the other remotes +can have duplicated files. + +Duplicated files cause problems with the syncing and you will see +messages in the log about duplicates. + +Use rclone dedupe to fix duplicated files. + +Note that this isn't just a problem with rclone, even Google Photos on +Android duplicates files on drive sometimes. + +Rclone appears to be re-copying files it shouldn't + +The most likely cause of this is the duplicated file issue above - run +rclone dedupe and check your logs for duplicate object or directory +messages. + +Making your own client_id + +When you use rclone with Google drive in its default configuration you +are using rclone's client_id. This is shared between all the rclone +users. There is a global rate limit on the number of queries per second +that each client_id can do set by Google. rclone already has a high +quota and I will continue to make sure it is high enough by contacting +Google. + +However you might find you get better performance making your own +client_id if you are a heavy user. Or you may not depending on exactly +how Google have been raising rclone's rate limit. + +Here is how to create your own Google Drive client ID for rclone: + +1. Log into the Google API Console with your Google account. It doesn't + matter what Google account you use. (It need not be the same account + as the Google Drive you want to access) + +2. Select a project or create a new project. + +3. Under "ENABLE APIS AND SERVICES" search for "Drive", and enable the + then "Google Drive API". + +4. Click "Credentials" in the left-side panel (not "Create + credentials", which opens the wizard), then "Create credentials", + then "OAuth client ID". It will prompt you to set the OAuth consent + screen product name, if you haven't set one already. + +5. Choose an application type of "other", and click "Create". (the + default name is fine) + +6. It will show you a client ID and client secret. Use these values in + rclone config to add a new remote or edit an existing remote. + +(Thanks to @balazer on github for these instructions.) + + +HTTP + +The HTTP remote is a read only remote for reading files of a webserver. +The webserver should provide file listings which rclone will read and +turn into a remote. This has been tested with common webservers such as +Apache/Nginx/Caddy and will likely work with file listings from most web +servers. (If it doesn't then please file an issue, or send a pull +request!) + +Paths are specified as remote: or remote:path/to/dir. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / FTP Connection + \ "ftp" + 7 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 8 / Google Drive + \ "drive" + 9 / Hubic + \ "hubic" + 10 / Local Disk + \ "local" + 11 / Microsoft OneDrive + \ "onedrive" + 12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 13 / SSH/SFTP Connection + \ "sftp" + 14 / Yandex Disk + \ "yandex" + 15 / http Connection + \ "http" + Storage> http + URL of http host to connect to + Choose a number from below, or type in your own value + 1 / Connect to example.com + \ "https://example.com" + url> https://beta.rclone.org + Remote config + -------------------- + [remote] + url = https://beta.rclone.org + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + Current remotes: + + Name Type + ==== ==== + remote http + + e) Edit existing remote + n) New remote + d) Delete remote + r) Rename remote + c) Copy remote + s) Set configuration password + q) Quit config + e/n/d/r/c/s/q> q + +This remote is called remote and can now be used like this + +See all the top level directories + + rclone lsd remote: + +List the contents of a directory + + rclone ls remote:directory + +Sync the remote directory to /home/local/directory, deleting any excess +files. + + rclone sync remote:directory /home/local/directory + +Read only + +This remote is read only - you can't upload files to an HTTP server. + +Modified time + +Most HTTP servers store time accurate to 1 second. + +Checksum + +No checksums are stored. + +Usage without a config file + +Since the http remote only has one config parameter it is easy to use +without a config file: + + rclone lsd --http-url https://beta.rclone.org :http: + + +Hubic + +Paths are specified as remote:path + +Paths are specified as remote:container (or remote: for the lsd +command.) You may put subdirectories in too, eg +remote:container/path/to/dir. + +The initial setup for Hubic involves getting a token from Hubic which +you need to do in your browser. rclone config walks you through it. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + n) New remote + s) Set configuration password + n/s> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" + 10 / Microsoft OneDrive + \ "onedrive" + 11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 12 / SSH/SFTP Connection + \ "sftp" + 13 / Yandex Disk + \ "yandex" + Storage> 8 + Hubic Client Id - leave blank normally. + client_id> + Hubic Client Secret - leave blank normally. + client_secret> + Remote config + Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine + y) Yes + n) No + y/n> y + If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth + Log in and authorize rclone for access + Waiting for code... + Got code + -------------------- + [remote] + client_id = + client_secret = + token = {"access_token":"XXXXXX"} + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Hubic. This only runs from the moment it opens +your browser to the moment you get back the verification code. This is +on http://127.0.0.1:53682/ and this it may require you to unblock it +temporarily if you are running a host firewall. + +Once configured you can then use rclone like this, + +List containers in the top level of your Hubic + + rclone lsd remote: + +List all the files in your Hubic + + rclone ls remote: + +To copy a local directory to an Hubic directory called backup + + rclone copy /home/source remote:backup + +If you want the directory to be visible in the official _Hubic browser_, +you need to copy your files to the default directory + + rclone copy /home/source remote:default/backup + +--fast-list + +This remote supports --fast-list which allows you to use fewer +transactions in exchange for more memory. See the rclone docs for more +details. + +Modified time + +The modified time is stored as metadata on the object as +X-Object-Meta-Mtime as floating point since the epoch accurate to 1 ns. + +This is a defacto standard (used in the official python-swiftclient +amongst others) for storing the modification time for an object. + +Note that Hubic wraps the Swift backend, so most of the properties of +are the same. + +Limitations + +This uses the normal OpenStack Swift mechanism to refresh the Swift API +credentials and ignores the expires field returned by the Hubic API. + +The Swift API doesn't return a correct MD5SUM for segmented files +(Dynamic or Static Large Objects) so rclone won't check or use the +MD5SUM for these. + + +Jottacloud + +Paths are specified as remote:path + +Paths may be as deep as required, eg remote:directory/subdirectory. + +To configure Jottacloud you will need to enter your username and +password and select a mountpoint. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Enter a string value. Press Enter for the default (""). + Choose a number from below, or type in your own value + [snip] + 13 / JottaCloud + \ "jottacloud" + [snip] + Storage> jottacloud + User Name + Enter a string value. Press Enter for the default (""). + user> user + Password. + y) Yes type in my own password + g) Generate random password + n) No leave this optional password blank + y/g/n> y + Enter the password: + password: + Confirm the password: + password: + The mountpoint to use. + Enter a string value. Press Enter for the default (""). + Choose a number from below, or type in your own value + 1 / Will be synced by the official client. + \ "Sync" + 2 / Archive + \ "Archive" + mountpoint> Archive + Remote config + -------------------- + [remote] + type = jottacloud + user = user + pass = *** ENCRYPTED *** + mountpoint = Archive + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +Once configured you can then use rclone like this, + +List directories in top level of your Jottacloud + + rclone lsd remote: + +List all the files in your Jottacloud + + rclone ls remote: + +To copy a local directory to an Jottacloud directory called backup + + rclone copy /home/source remote:backup + +Modified time and hashes + +Jottacloud allows modification times to be set on objects accurate to 1 +second. These will be used to detect whether objects need syncing or +not. + +Jottacloud supports MD5 type hashes, so you can use the --checksum flag. + +Note that Jottacloud requires the MD5 hash before upload so if the +source does not have an MD5 checksum then the file will be cached +temporarily on disk (wherever the TMPDIR environment variable points to) +before it is uploaded. Small files will be cached in memory - see the +--jottacloud-md5-memory-limit flag. + +Deleting files + +Any files you delete with rclone will end up in the trash. Due to a lack +of API documentation emptying the trash is currently only possible via +the Jottacloud website. + +Versions + +Jottacloud supports file versioning. When rclone uploads a new version +of a file it creates a new version of it. Currently rclone only supports +retrieving the current version but older versions can be accessed via +the Jottacloud Website. + +Limitations + +Note that Jottacloud is case insensitive so you can't have a file called +"Hello.doc" and one called "hello.doc". + +There are quite a few characters that can't be in Jottacloud file names. +Rclone will map these names to and from an identical looking unicode +equivalent. For example if a file has a ? in it will be mapped to ？ +instead. + +Jottacloud only supports filenames up to 255 characters in length. + +Specific options + +Here are the command line options specific to this cloud storage system. + +--jottacloud-md5-memory-limit SizeSuffix + +Files bigger than this will be cached on disk to calculate the MD5 if +required. (default 10M) + +Troubleshooting + +Jottacloud exhibits some inconsistent behaviours regarding deleted files +and folders which may cause Copy, Move and DirMove operations to +previously deleted paths to fail. Emptying the trash should help in such +cases. + + +Mega + +Mega is a cloud storage and file hosting service known for its security +feature where all files are encrypted locally before they are uploaded. +This prevents anyone (including employees of Mega) from accessing the +files without knowledge of the key used for encryption. + +This is an rclone backend for Mega which supports the file transfer +features of Mega using the same client side encryption. + +Paths are specified as remote:path + +Paths may be as deep as required, eg remote:directory/subdirectory. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" + [snip] + 14 / Mega + \ "mega" + [snip] + 23 / http Connection + \ "http" + Storage> mega + User name + user> you@example.com + Password. + y) Yes type in my own password + g) Generate random password + n) No leave this optional password blank + y/g/n> y + Enter the password: + password: + Confirm the password: + password: + Remote config + -------------------- + [remote] + type = mega + user = you@example.com + pass = *** ENCRYPTED *** + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +Once configured you can then use rclone like this, + +List directories in top level of your Mega + + rclone lsd remote: + +List all the files in your Mega + + rclone ls remote: + +To copy a local directory to an Mega directory called backup + + rclone copy /home/source remote:backup + +Modified time and hashes + +Mega does not support modification times or hashes yet. + +Duplicated files + +Mega can have two files with exactly the same name and path (unlike a +normal file system). + +Duplicated files cause problems with the syncing and you will see +messages in the log about duplicates. + +Use rclone dedupe to fix duplicated files. + +Specific options + +Here are the command line options specific to this cloud storage system. + +--mega-debug + +If this flag is set (along with -vv) it will print further debugging +information from the mega backend. + +--mega-hard-delete + +Normally the mega backend will put all deletions into the trash rather +than permanently deleting them. If you specify this flag (or set it in +the advanced config) then rclone will permanently delete objects +instead. + +Limitations + +This backend uses the go-mega go library which is an opensource go +library implementing the Mega API. There doesn't appear to be any +documentation for the mega protocol beyond the mega C++ SDK source code +so there are likely quite a few errors still remaining in this library. + +Mega allows duplicate files which may confuse rclone. + + +Microsoft Azure Blob Storage + +Paths are specified as remote:container (or remote: for the lsd +command.) You may put subdirectories in too, eg +remote:container/path/to/dir. + +Here is an example of making a Microsoft Azure Blob Storage +configuration. For a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Box + \ "box" + 5 / Dropbox + \ "dropbox" + 6 / Encrypt/Decrypt a remote + \ "crypt" + 7 / FTP Connection + \ "ftp" + 8 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 9 / Google Drive + \ "drive" + 10 / Hubic + \ "hubic" + 11 / Local Disk + \ "local" + 12 / Microsoft Azure Blob Storage + \ "azureblob" + 13 / Microsoft OneDrive + \ "onedrive" + 14 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 15 / SSH/SFTP Connection + \ "sftp" + 16 / Yandex Disk + \ "yandex" + 17 / http Connection + \ "http" + Storage> azureblob + Storage Account Name + account> account_name + Storage Account Key + key> base64encodedkey== + Endpoint for the service - leave blank normally. + endpoint> + Remote config + -------------------- + [remote] + account = account_name + key = base64encodedkey== + endpoint = + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +See all containers + + rclone lsd remote: + +Make a new container + + rclone mkdir remote:container + +List the contents of a container + + rclone ls remote:container + +Sync /home/local/directory to the remote container, deleting any excess +files in the container. + + rclone sync /home/local/directory remote:container + +--fast-list + +This remote supports --fast-list which allows you to use fewer +transactions in exchange for more memory. See the rclone docs for more +details. + +Modified time + +The modified time is stored as metadata on the object with the mtime +key. It is stored using RFC3339 Format time with nanosecond precision. +The metadata is supplied during directory listings so there is no +overhead to using it. + +Hashes + +MD5 hashes are stored with blobs. However blobs that were uploaded in +chunks only have an MD5 if the source remote was capable of MD5 hashes, +eg the local disk. + +Authenticating with Azure Blob Storage + +Rclone has 3 ways of authenticating with Azure Blob Storage: + +Account and Key + +This is the most straight forward and least flexible way. Just fill in +the account and key lines and leave the rest blank. + +SAS URL + +This can be an account level SAS URL or container level SAS URL + +To use it leave account, key blank and fill in sas_url. + +Account level SAS URL or container level SAS URL can be obtained from +Azure portal or Azure Storage Explorer. To get a container level SAS URL +right click on a container in the Azure Blob explorer in the Azure +portal. + +If You use container level SAS URL, rclone operations are permitted only +on particular container, eg + + rclone ls azureblob:container or rclone ls azureblob: + +Since container name already exists in SAS URL, you can leave it empty +as well. + +However these will not work + + rclone lsd azureblob: + rclone ls azureblob:othercontainer + +This would be useful for temporarily allowing third parties access to a +single container or putting credentials into an untrusted environment. + +Multipart uploads + +Rclone supports multipart uploads with Azure Blob storage. Files bigger +than 256MB will be uploaded using chunked upload by default. + +The files will be uploaded in parallel in 4MB chunks (by default). Note +that these chunks are buffered in memory and there may be up to +--transfers of them being uploaded at once. + +Files can't be split into more than 50,000 chunks so by default, so the +largest file that can be uploaded with 4MB chunk size is 195GB. Above +this rclone will double the chunk size until it creates less than 50,000 +chunks. By default this will mean a maximum file size of 3.2TB can be +uploaded. This can be raised to 5TB using --azureblob-chunk-size 100M. + +Note that rclone doesn't commit the block list until the end of the +upload which means that there is a limit of 9.5TB of multipart uploads +in progress as Azure won't allow more than that amount of uncommitted +blocks. + +Specific options + +Here are the command line options specific to this cloud storage system. + +--azureblob-upload-cutoff=SIZE + +Cutoff for switching to chunked upload - must be <= 256MB. The default +is 256MB. + +--azureblob-chunk-size=SIZE + +Upload chunk size. Default 4MB. Note that this is stored in memory and +there may be up to --transfers chunks stored at once in memory. This can +be at most 100MB. + +--azureblob-access-tier=Hot/Cool/Archive + +Azure storage supports blob tiering, you can configure tier in advanced +settings or supply flag while performing data transfer operations. If +there is no access tier specified, rclone doesn't apply any tier. rclone +performs Set Tier operation on blobs while uploading, if objects are not +modified, specifying access tier to new one will have no effect. If +blobs are in archive tier at remote, trying to perform data transfer +operations from remote will not be allowed. User should first restore by +tiering blob to Hot or Cool. + +Limitations + +MD5 sums are only uploaded with chunked files if the source has an MD5 +sum. This will always be the case for a local to azure copy. + + +Microsoft OneDrive + +Paths are specified as remote:path + +Paths may be as deep as required, eg remote:directory/subdirectory. + +The initial setup for OneDrive involves getting a token from Microsoft +which you need to do in your browser. rclone config walks you through +it. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + s) Set configuration password + n/s> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" + 10 / Microsoft OneDrive + \ "onedrive" + 11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 12 / SSH/SFTP Connection + \ "sftp" + 13 / Yandex Disk + \ "yandex" + Storage> 10 + Microsoft App Client Id - leave blank normally. + client_id> + Microsoft App Client Secret - leave blank normally. + client_secret> + Remote config + Choose OneDrive account type? + * Say b for a OneDrive business account + * Say p for a personal OneDrive account + b) Business + p) Personal + b/p> p + Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine + y) Yes + n) No + y/n> y + If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth + Log in and authorize rclone for access + Waiting for code... + Got code + -------------------- + [remote] + client_id = + client_secret = + token = {"access_token":"XXXXXX"} + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Microsoft. This only runs from the moment it +opens your browser to the moment you get back the verification code. +This is on http://127.0.0.1:53682/ and this it may require you to +unblock it temporarily if you are running a host firewall. + +Once configured you can then use rclone like this, + +List directories in top level of your OneDrive + + rclone lsd remote: + +List all the files in your OneDrive + + rclone ls remote: + +To copy a local directory to an OneDrive directory called backup + + rclone copy /home/source remote:backup + +OneDrive for Business + +There is additional support for OneDrive for Business. Select "b" when +ask + + Choose OneDrive account type? + * Say b for a OneDrive business account + * Say p for a personal OneDrive account + b) Business + p) Personal + b/p> + +After that rclone requires an authentication of your account. The +application will first authenticate your account, then query the +OneDrive resource URL and do a second (silent) authentication for this +resource URL. + +Modified time and hashes + +OneDrive allows modification times to be set on objects accurate to 1 +second. These will be used to detect whether objects need syncing or +not. + +OneDrive personal supports SHA1 type hashes. OneDrive for business and +Sharepoint Server support QuickXorHash. + +For all types of OneDrive you can use the --checksum flag. + +Deleting files + +Any files you delete with rclone will end up in the trash. Microsoft +doesn't provide an API to permanently delete files, nor to empty the +trash, so you will have to do that with one of Microsoft's apps or via +the OneDrive website. + +Specific options + +Here are the command line options specific to this cloud storage system. + +--onedrive-chunk-size=SIZE + +Above this size files will be chunked - must be multiple of 320k. The +default is 10MB. Note that the chunks will be buffered into memory. + +Limitations + +Note that OneDrive is case insensitive so you can't have a file called +"Hello.doc" and one called "hello.doc". + +There are quite a few characters that can't be in OneDrive file names. +These can't occur on Windows platforms, but on non-Windows platforms +they are common. Rclone will map these names to and from an identical +looking unicode equivalent. For example if a file has a ? in it will be +mapped to ？ instead. + +The largest allowed file size is 10GiB (10,737,418,240 bytes). + +Versioning issue + +Every change in OneDrive causes the service to create a new version. +This counts against a users quota. For example changing the modification +time of a file creates a second version, so the file is using twice the +space. + +The copy is the only rclone command affected by this as we copy the file +and then afterwards set the modification time to match the source file. + +User Weropol has found a method to disable versioning on OneDrive + +1. Open the settings menu by clicking on the gear symbol at the top of + the OneDrive Business page. +2. Click Site settings. +3. Once on the Site settings page, navigate to Site Administration > + Site libraries and lists. +4. Click Customize "Documents". +5. Click General Settings > Versioning Settings. +6. Under Document Version History select the option No versioning. + Note: This will disable the creation of new file versions, but will + not remove any previous versions. Your documents are safe. +7. Apply the changes by clicking OK. +8. Use rclone to upload or modify files. (I also use the + --no-update-modtime flag) +9. Restore the versioning settings after using rclone. (Optional) + +Troubleshooting + + Error: access_denied + Code: AADSTS65005 + Description: Using application 'rclone' is currently not supported for your organization [YOUR_ORGANIZATION] because it is in an unmanaged state. An administrator needs to claim ownership of the company by DNS validation of [YOUR_ORGANIZATION] before the application rclone can be provisioned. + +This means that rclone can't use the OneDrive for Business API with your +account. You can't do much about it, maybe write an email to your +admins. + +However, there are other ways to interact with your OneDrive account. +Have a look at the webdav backend: https://rclone.org/webdav/#sharepoint + + +OpenDrive + +Paths are specified as remote:path + +Paths may be as deep as required, eg remote:directory/subdirectory. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + n) New remote + d) Delete remote + q) Quit config + e/n/d/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" + 10 / OpenDrive + \ "opendrive" + 11 / Microsoft OneDrive + \ "onedrive" + 12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 13 / SSH/SFTP Connection + \ "sftp" + 14 / Yandex Disk + \ "yandex" + Storage> 10 + Username + username> + Password + y) Yes type in my own password + g) Generate random password + y/g> y + Enter the password: + password: + Confirm the password: + password: + -------------------- + [remote] + username = + password = *** ENCRYPTED *** + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +List directories in top level of your OpenDrive + + rclone lsd remote: + +List all the files in your OpenDrive + + rclone ls remote: + +To copy a local directory to an OpenDrive directory called backup + + rclone copy /home/source remote:backup + +Modified time and MD5SUMs + +OpenDrive allows modification times to be set on objects accurate to 1 +second. These will be used to detect whether objects need syncing or +not. + +Deleting files + +Any files you delete with rclone will end up in the trash. Amazon don't +provide an API to permanently delete files, nor to empty the trash, so +you will have to do that with one of Amazon's apps or via the OpenDrive +website. As of November 17, 2016, files are automatically deleted by +Amazon from the trash after 30 days. + +Limitations + +Note that OpenDrive is case insensitive so you can't have a file called +"Hello.doc" and one called "hello.doc". + +There are quite a few characters that can't be in OpenDrive file names. +These can't occur on Windows platforms, but on non-Windows platforms +they are common. Rclone will map these names to and from an identical +looking unicode equivalent. For example if a file has a ? in it will be +mapped to ？ instead. + + +QingStor + +Paths are specified as remote:bucket (or remote: for the lsd command.) +You may put subdirectories in too, eg remote:bucket/path/to/dir. + +Here is an example of making an QingStor configuration. First run + + rclone config + +This will guide you through an interactive setup process. + + No remotes found - make a new one + n) New remote + r) Rename remote + c) Copy remote + s) Set configuration password + q) Quit config + n/r/c/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / FTP Connection + \ "ftp" + 7 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 8 / Google Drive + \ "drive" + 9 / Hubic + \ "hubic" + 10 / Local Disk + \ "local" + 11 / Microsoft OneDrive + \ "onedrive" + 12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 13 / QingStor Object Storage + \ "qingstor" + 14 / SSH/SFTP Connection + \ "sftp" + 15 / Yandex Disk + \ "yandex" + Storage> 13 + Get QingStor credentials from runtime. Only applies if access_key_id and secret_access_key is blank. + Choose a number from below, or type in your own value + 1 / Enter QingStor credentials in the next step + \ "false" + 2 / Get QingStor credentials from the environment (env vars or IAM) + \ "true" + env_auth> 1 + QingStor Access Key ID - leave blank for anonymous access or runtime credentials. + access_key_id> access_key + QingStor Secret Access Key (password) - leave blank for anonymous access or runtime credentials. + secret_access_key> secret_key + Enter a endpoint URL to connection QingStor API. + Leave blank will use the default value "https://qingstor.com:443" + endpoint> + Zone connect to. Default is "pek3a". + Choose a number from below, or type in your own value + / The Beijing (China) Three Zone + 1 | Needs location constraint pek3a. + \ "pek3a" + / The Shanghai (China) First Zone + 2 | Needs location constraint sh1a. + \ "sh1a" + zone> 1 + Number of connnection retry. + Leave blank will use the default value "3". + connection_retries> + Remote config + -------------------- + [remote] + env_auth = false + access_key_id = access_key + secret_access_key = secret_key + endpoint = + zone = pek3a + connection_retries = + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +This remote is called remote and can now be used like this + +See all buckets + + rclone lsd remote: + +Make a new bucket + + rclone mkdir remote:bucket + +List the contents of a bucket + + rclone ls remote:bucket + +Sync /home/local/directory to the remote bucket, deleting any excess +files in the bucket. + + rclone sync /home/local/directory remote:bucket + +--fast-list + +This remote supports --fast-list which allows you to use fewer +transactions in exchange for more memory. See the rclone docs for more +details. + +Multipart uploads + +rclone supports multipart uploads with QingStor which means that it can +upload files bigger than 5GB. Note that files uploaded with multipart +upload don't have an MD5SUM. + +Buckets and Zone + +With QingStor you can list buckets (rclone lsd) using any zone, but you +can only access the content of a bucket from the zone it was created in. +If you attempt to access a bucket from the wrong zone, you will get an +error, incorrect zone, the bucket is not in 'XXX' zone. + +Authentication + +There are two ways to supply rclone with a set of QingStor credentials. +In order of precedence: + +- Directly in the rclone configuration file (as configured by + rclone config) +- set access_key_id and secret_access_key +- Runtime configuration: +- set env_auth to true in the config file +- Exporting the following environment variables before running rclone + - Access Key ID: QS_ACCESS_KEY_ID or QS_ACCESS_KEY + - Secret Access Key: QS_SECRET_ACCESS_KEY or QS_SECRET_KEY + + +Swift + +Swift refers to Openstack Object Storage. Commercial implementations of +that being: + +- Rackspace Cloud Files +- Memset Memstore +- OVH Object Storage +- Oracle Cloud Storage +- IBM Bluemix Cloud ObjectStorage Swift + +Paths are specified as remote:container (or remote: for the lsd +command.) You may put subdirectories in too, eg +remote:container/path/to/dir. + +Here is an example of making a swift configuration. First run + + rclone config + +This will guide you through an interactive setup process. + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Box + \ "box" + 5 / Cache a remote + \ "cache" + 6 / Dropbox + \ "dropbox" + 7 / Encrypt/Decrypt a remote + \ "crypt" + 8 / FTP Connection + \ "ftp" + 9 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 10 / Google Drive + \ "drive" + 11 / Hubic + \ "hubic" + 12 / Local Disk + \ "local" + 13 / Microsoft Azure Blob Storage + \ "azureblob" + 14 / Microsoft OneDrive + \ "onedrive" + 15 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 16 / Pcloud + \ "pcloud" + 17 / QingCloud Object Storage + \ "qingstor" + 18 / SSH/SFTP Connection + \ "sftp" + 19 / Webdav + \ "webdav" + 20 / Yandex Disk + \ "yandex" + 21 / http Connection + \ "http" + Storage> swift + Get swift credentials from environment variables in standard OpenStack form. + Choose a number from below, or type in your own value + 1 / Enter swift credentials in the next step + \ "false" + 2 / Get swift credentials from environment vars. Leave other fields blank if using this. + \ "true" + env_auth> true + User name to log in (OS_USERNAME). + user> + API key or password (OS_PASSWORD). + key> + Authentication URL for server (OS_AUTH_URL). + Choose a number from below, or type in your own value + 1 / Rackspace US + \ "https://auth.api.rackspacecloud.com/v1.0" + 2 / Rackspace UK + \ "https://lon.auth.api.rackspacecloud.com/v1.0" + 3 / Rackspace v2 + \ "https://identity.api.rackspacecloud.com/v2.0" + 4 / Memset Memstore UK + \ "https://auth.storage.memset.com/v1.0" + 5 / Memset Memstore UK v2 + \ "https://auth.storage.memset.com/v2.0" + 6 / OVH + \ "https://auth.cloud.ovh.net/v2.0" + auth> + User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID). + user_id> + User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME) + domain> + Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME) + tenant> + Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID) + tenant_id> + Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME) + tenant_domain> + Region name - optional (OS_REGION_NAME) + region> + Storage URL - optional (OS_STORAGE_URL) + storage_url> + Auth Token from alternate authentication - optional (OS_AUTH_TOKEN) + auth_token> + AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION) + auth_version> + Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) + Choose a number from below, or type in your own value + 1 / Public (default, choose this if not sure) + \ "public" + 2 / Internal (use internal service net) + \ "internal" + 3 / Admin + \ "admin" + endpoint_type> + Remote config + -------------------- + [test] + env_auth = true + user = + key = + auth = + user_id = + domain = + tenant = + tenant_id = + tenant_domain = + region = + storage_url = + auth_token = + auth_version = + endpoint_type = + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +This remote is called remote and can now be used like this + +See all containers + + rclone lsd remote: + +Make a new container + + rclone mkdir remote:container + +List the contents of a container + + rclone ls remote:container + +Sync /home/local/directory to the remote container, deleting any excess +files in the container. + + rclone sync /home/local/directory remote:container + +Configuration from an Openstack credentials file + +An Opentstack credentials file typically looks something something like +this (without the comments) + + export OS_AUTH_URL=https://a.provider.net/v2.0 + export OS_TENANT_ID=ffffffffffffffffffffffffffffffff + export OS_TENANT_NAME="1234567890123456" + export OS_USERNAME="123abc567xy" + echo "Please enter your OpenStack Password: " + read -sr OS_PASSWORD_INPUT + export OS_PASSWORD=$OS_PASSWORD_INPUT + export OS_REGION_NAME="SBG1" + if [ -z "$OS_REGION_NAME" ]; then unset OS_REGION_NAME; fi + +The config file needs to look something like this where $OS_USERNAME +represents the value of the OS_USERNAME variable - 123abc567xy in the +example above. + + [remote] + type = swift + user = $OS_USERNAME + key = $OS_PASSWORD + auth = $OS_AUTH_URL + tenant = $OS_TENANT_NAME + +Note that you may (or may not) need to set region too - try without +first. + +Configuration from the environment + +If you prefer you can configure rclone to use swift using a standard set +of OpenStack environment variables. + +When you run through the config, make sure you choose true for env_auth +and leave everything else blank. + +rclone will then set any empty config parameters from the environment +using standard OpenStack environment variables. There is a list of the +variables in the docs for the swift library. + +Using an alternate authentication method + +If your OpenStack installation uses a non-standard authentication method +that might not be yet supported by rclone or the underlying swift +library, you can authenticate externally (e.g. calling manually the +openstack commands to get a token). Then, you just need to pass the two +configuration variables auth_token and storage_url. If they are both +provided, the other variables are ignored. rclone will not try to +authenticate but instead assume it is already authenticated and use +these two variables to access the OpenStack installation. + +Using rclone without a config file + +You can use rclone with swift without a config file, if desired, like +this: + + source openstack-credentials-file + export RCLONE_CONFIG_MYREMOTE_TYPE=swift + export RCLONE_CONFIG_MYREMOTE_ENV_AUTH=true + rclone lsd myremote: + +--fast-list + +This remote supports --fast-list which allows you to use fewer +transactions in exchange for more memory. See the rclone docs for more +details. + +--update and --use-server-modtime + +As noted below, the modified time is stored on metadata on the object. +It is used by default for all operations that require checking the time +a file was last updated. It allows rclone to treat the remote more like +a true filesystem, but it is inefficient because it requires an extra +API call to retrieve the metadata. + +For many operations, the time the object was last uploaded to the remote +is sufficient to determine if it is "dirty". By using --update along +with --use-server-modtime, you can avoid the extra API call and simply +upload files whose local modtime is newer than the time it was last +uploaded. + +Specific options + +Here are the command line options specific to this cloud storage system. + +--swift-storage-policy=STRING + +Apply the specified storage policy when creating a new container. The +policy cannot be changed afterwards. The allowed configuration values +and their meaning depend on your Swift storage provider. + +--swift-chunk-size=SIZE + +Above this size files will be chunked into a _segments container. The +default for this is 5GB which is its maximum value. + +Modified time + +The modified time is stored as metadata on the object as +X-Object-Meta-Mtime as floating point since the epoch accurate to 1 ns. + +This is a defacto standard (used in the official python-swiftclient +amongst others) for storing the modification time for an object. + +Limitations + +The Swift API doesn't return a correct MD5SUM for segmented files +(Dynamic or Static Large Objects) so rclone won't check or use the +MD5SUM for these. + +Troubleshooting + +Rclone gives Failed to create file system for "remote:": Bad Request + +Due to an oddity of the underlying swift library, it gives a "Bad +Request" error rather than a more sensible error when the authentication +fails for Swift. + +So this most likely means your username / password is wrong. You can +investigate further with the --dump-bodies flag. + +This may also be caused by specifying the region when you shouldn't have +(eg OVH). + +Rclone gives Failed to create file system: Response didn't have storage storage url and auth token + +This is most likely caused by forgetting to specify your tenant when +setting up a swift remote. + + +pCloud + +Paths are specified as remote:path + +Paths may be as deep as required, eg remote:directory/subdirectory. + +The initial setup for pCloud involves getting a token from pCloud which +you need to do in your browser. rclone config walks you through it. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Box + \ "box" + 5 / Dropbox + \ "dropbox" + 6 / Encrypt/Decrypt a remote + \ "crypt" + 7 / FTP Connection + \ "ftp" + 8 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 9 / Google Drive + \ "drive" + 10 / Hubic + \ "hubic" + 11 / Local Disk + \ "local" + 12 / Microsoft Azure Blob Storage + \ "azureblob" + 13 / Microsoft OneDrive + \ "onedrive" + 14 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 15 / Pcloud + \ "pcloud" + 16 / QingCloud Object Storage + \ "qingstor" + 17 / SSH/SFTP Connection + \ "sftp" + 18 / Yandex Disk + \ "yandex" + 19 / http Connection + \ "http" + Storage> pcloud + Pcloud App Client Id - leave blank normally. + client_id> + Pcloud App Client Secret - leave blank normally. + client_secret> + Remote config + Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine + y) Yes + n) No + y/n> y + If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth + Log in and authorize rclone for access + Waiting for code... + Got code + -------------------- + [remote] + client_id = + client_secret = + token = {"access_token":"XXX","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"} + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from pCloud. This only runs from the moment it opens +your browser to the moment you get back the verification code. This is +on http://127.0.0.1:53682/ and this it may require you to unblock it +temporarily if you are running a host firewall. + +Once configured you can then use rclone like this, + +List directories in top level of your pCloud + + rclone lsd remote: + +List all the files in your pCloud + + rclone ls remote: + +To copy a local directory to an pCloud directory called backup + + rclone copy /home/source remote:backup + +Modified time and hashes + +pCloud allows modification times to be set on objects accurate to 1 +second. These will be used to detect whether objects need syncing or +not. In order to set a Modification time pCloud requires the object be +re-uploaded. + +pCloud supports MD5 and SHA1 type hashes, so you can use the --checksum +flag. + +Deleting files + +Deleted files will be moved to the trash. Your subscription level will +determine how long items stay in the trash. rclone cleanup can be used +to empty the trash. + + +SFTP + +SFTP is the Secure (or SSH) File Transfer Protocol. + +SFTP runs over SSH v2 and is installed as standard with most modern SSH +installations. + +Paths are specified as remote:path. If the path does not begin with a / +it is relative to the home directory of the user. An empty path remote: +refers to the user's home directory. + +Note that some SFTP servers will need the leading / - Synology is a good +example of this. + +Here is an example of making an SFTP configuration. First run + + rclone config + +This will guide you through an interactive setup process. + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / FTP Connection + \ "ftp" + 7 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 8 / Google Drive + \ "drive" + 9 / Hubic + \ "hubic" + 10 / Local Disk + \ "local" + 11 / Microsoft OneDrive + \ "onedrive" + 12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 13 / SSH/SFTP Connection + \ "sftp" + 14 / Yandex Disk + \ "yandex" + 15 / http Connection + \ "http" + Storage> sftp + SSH host to connect to + Choose a number from below, or type in your own value + 1 / Connect to example.com + \ "example.com" + host> example.com + SSH username, leave blank for current username, ncw + user> sftpuser + SSH port, leave blank to use default (22) + port> + SSH password, leave blank to use ssh-agent. + y) Yes type in my own password + g) Generate random password + n) No leave this optional password blank + y/g/n> n + Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent. + key_file> + Remote config + -------------------- + [remote] + host = example.com + user = sftpuser + port = + pass = + key_file = + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +This remote is called remote and can now be used like this: + +See all directories in the home directory + + rclone lsd remote: + +Make a new directory + + rclone mkdir remote:path/to/directory + +List the contents of a directory + + rclone ls remote:path/to/directory + +Sync /home/local/directory to the remote directory, deleting any excess +files in the directory. + + rclone sync /home/local/directory remote:directory + +SSH Authentication + +The SFTP remote supports three authentication methods: + +- Password +- Key file +- ssh-agent + +Key files should be unencrypted PEM-encoded private key files. For +instance /home/$USER/.ssh/id_rsa. + +If you don't specify pass or key_file then rclone will attempt to +contact an ssh-agent. + +If you set the --sftp-ask-password option, rclone will prompt for a +password when needed and no password has been configured. + +ssh-agent on macOS + +Note that there seem to be various problems with using an ssh-agent on +macOS due to recent changes in the OS. The most effective work-around +seems to be to start an ssh-agent in each session, eg + + eval `ssh-agent -s` && ssh-add -A + +And then at the end of the session + + eval `ssh-agent -k` + +These commands can be used in scripts of course. + +Specific options + +Here are the command line options specific to this remote. + +--sftp-ask-password + +Ask for the SFTP password if needed when no password has been +configured. + +--ssh-path-override + +Override path used by SSH connection. Allows checksum calculation when +SFTP and SSH paths are different. This issue affects among others +Synology NAS boxes. + +Shared folders can be found in directories representing volumes + + rclone sync /home/local/directory remote:/directory --ssh-path-override /volume2/directory + +Home directory can be found in a shared folder called homes + + rclone sync /home/local/directory remote:/home/directory --ssh-path-override /volume1/homes/USER/directory + +Modified time + +Modified times are stored on the server to 1 second precision. + +Modified times are used in syncing and are fully supported. + +Some SFTP servers disable setting/modifying the file modification time +after upload (for example, certain configurations of ProFTPd with +mod_sftp). If you are using one of these servers, you can set the option +set_modtime = false in your RClone backend configuration to disable this +behaviour. + +Limitations + +SFTP supports checksums if the same login has shell access and md5sum or +sha1sum as well as echo are in the remote's PATH. This remote +checksumming (file hashing) is recommended and enabled by default. +Disabling the checksumming may be required if you are connecting to SFTP +servers which are not under your control, and to which the execution of +remote commands is prohibited. Set the configuration option +disable_hashcheck to true to disable checksumming. + +Note that some SFTP servers (eg Synology) the paths are different for +SSH and SFTP so the hashes can't be calculated properly. For them using +disable_hashcheck is a good idea. + +The only ssh agent supported under Windows is Putty's pageant. + +The Go SSH library disables the use of the aes128-cbc cipher by default, +due to security concerns. This can be re-enabled on a per-connection +basis by setting the use_insecure_cipher setting in the configuration +file to true. Further details on the insecurity of this cipher can be +found [in this paper] (http://www.isg.rhul.ac.uk/~kp/SandPfinal.pdf). + +SFTP isn't supported under plan9 until this issue is fixed. + +Note that since SFTP isn't HTTP based the following flags don't work +with it: --dump-headers, --dump-bodies, --dump-auth + +Note that --timeout isn't supported (but --contimeout is). + + +WebDAV + +Paths are specified as remote:path + +Paths may be as deep as required, eg remote:directory/subdirectory. + +To configure the WebDAV remote you will need to have a URL for it, and a +username and password. If you know what kind of system you are +connecting to then rclone can enable extra features. + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + [snip] + 22 / Webdav + \ "webdav" + [snip] + Storage> webdav + URL of http host to connect to + Choose a number from below, or type in your own value + 1 / Connect to example.com + \ "https://example.com" + url> https://example.com/remote.php/webdav/ + Name of the Webdav site/service/software you are using + Choose a number from below, or type in your own value + 1 / Nextcloud + \ "nextcloud" + 2 / Owncloud + \ "owncloud" + 3 / Sharepoint + \ "sharepoint" + 4 / Other site/service or software + \ "other" + vendor> 1 + User name + user> user + Password. + y) Yes type in my own password + g) Generate random password + n) No leave this optional password blank + y/g/n> y + Enter the password: + password: + Confirm the password: + password: + Bearer token instead of user/pass (eg a Macaroon) + bearer_token> + Remote config + -------------------- + [remote] + type = webdav + url = https://example.com/remote.php/webdav/ + vendor = nextcloud + user = user + pass = *** ENCRYPTED *** + bearer_token = + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +Once configured you can then use rclone like this, + +List directories in top level of your WebDAV + + rclone lsd remote: + +List all the files in your WebDAV + + rclone ls remote: + +To copy a local directory to an WebDAV directory called backup + + rclone copy /home/source remote:backup + +Modified time and hashes + +Plain WebDAV does not support modified times. However when used with +Owncloud or Nextcloud rclone will support modified times. + +Hashes are not supported. + + +Provider notes + +See below for notes on specific providers. + +Owncloud + +Click on the settings cog in the bottom right of the page and this will +show the WebDAV URL that rclone needs in the config step. It will look +something like https://example.com/remote.php/webdav/. + +Owncloud supports modified times using the X-OC-Mtime header. + +Nextcloud + +This is configured in an identical way to Owncloud. Note that Nextcloud +does not support streaming of files (rcat) whereas Owncloud does. This +may be fixed in the future. + +Put.io + +put.io can be accessed in a read only way using webdav. + +Configure the url as https://webdav.put.io and use your normal account +username and password for user and pass. Set the vendor to other. + +Your config file should end up looking like this: + + [putio] + type = webdav + url = https://webdav.put.io + vendor = other + user = YourUserName + pass = encryptedpassword + +If you are using put.io with rclone mount then use the --read-only flag +to signal to the OS that it can't write to the mount. + +For more help see the put.io webdav docs. + +Sharepoint + +Rclone can be used with Sharepoint provided by OneDrive for Business or +Office365 Education Accounts. This feature is only needed for a few of +these Accounts, mostly Office365 Education ones. These accounts are +sometimes not verified by the domain owner github#1975 + +This means that these accounts can't be added using the official API +(other Accounts should work with the "onedrive" option). However, it is +possible to access them using webdav. + +To use a sharepoint remote with rclone, add it like this: First, you +need to get your remote's URL: + +- Go here to open your OneDrive or to sign in +- Now take a look at your address bar, the URL should look like this: + https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/_layouts/15/onedrive.aspx + +You'll only need this URL upto the email address. After that, you'll +most likely want to add "/Documents". That subdirectory contains the +actual data stored on your OneDrive. + +Add the remote to rclone like this: Configure the url as +https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/Documents +and use your normal account email and password for user and pass. If you +have 2FA enabled, you have to generate an app password. Set the vendor +to sharepoint. + +Your config file should look like this: + + [sharepoint] + type = webdav + url = https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/Documents + vendor = other + user = YourEmailAddress + pass = encryptedpassword + +dCache + +dCache is a storage system with WebDAV doors that support, beside basic +and x509, authentication with Macaroons (bearer tokens). + +Configure as normal using the other type. Don't enter a username or +password, instead enter your Macaroon as the bearer_token. + +The config will end up looking something like this. + + [dcache] + type = webdav + url = https://dcache... + vendor = other + user = + pass = + bearer_token = your-macaroon + +There is a script that obtains a Macaroon from a dCache WebDAV endpoint, +and creates an rclone config file. + + +Yandex Disk + +Yandex Disk is a cloud storage solution created by Yandex. + +Yandex paths may be as deep as required, eg +remote:directory/subdirectory. + +Here is an example of making a yandex configuration. First run + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + s) Set configuration password + n/s> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" + 10 / Microsoft OneDrive + \ "onedrive" + 11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" + 12 / SSH/SFTP Connection + \ "sftp" + 13 / Yandex Disk + \ "yandex" + Storage> 13 + Yandex Client Id - leave blank normally. + client_id> + Yandex Client Secret - leave blank normally. + client_secret> + Remote config + Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine + y) Yes + n) No + y/n> y + If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth + Log in and authorize rclone for access + Waiting for code... + Got code + -------------------- + [remote] + client_id = + client_secret = + token = {"access_token":"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","token_type":"bearer","expiry":"2016-12-29T12:27:11.362788025Z"} + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Yandex Disk. This only runs from the moment it +opens your browser to the moment you get back the verification code. +This is on http://127.0.0.1:53682/ and this it may require you to +unblock it temporarily if you are running a host firewall. + +Once configured you can then use rclone like this, + +See top level directories + + rclone lsd remote: + +Make a new directory + + rclone mkdir remote:directory + +List the contents of a directory + + rclone ls remote:directory + +Sync /home/local/directory to the remote path, deleting any excess files +in the path. + + rclone sync /home/local/directory remote:directory + +--fast-list + +This remote supports --fast-list which allows you to use fewer +transactions in exchange for more memory. See the rclone docs for more +details. + +Modified time + +Modified times are supported and are stored accurate to 1 ns in custom +metadata called rclone_modified in RFC3339 with nanoseconds format. + +MD5 checksums + +MD5 checksums are natively supported by Yandex Disk. + +Emptying Trash + +If you wish to empty your trash you can use the rclone cleanup remote: +command which will permanently delete all your trashed files. This +command does not take any path arguments. + + +Local Filesystem + +Local paths are specified as normal filesystem paths, eg +/path/to/wherever, so + + rclone sync /home/source /tmp/destination + +Will sync /home/source to /tmp/destination + +These can be configured into the config file for consistencies sake, but +it is probably easier not to. + +Modified time + +Rclone reads and writes the modified time using an accuracy determined +by the OS. Typically this is 1ns on Linux, 10 ns on Windows and 1 Second +on OS X. + +Filenames + +Filenames are expected to be encoded in UTF-8 on disk. This is the +normal case for Windows and OS X. + +There is a bit more uncertainty in the Linux world, but new +distributions will have UTF-8 encoded files names. If you are using an +old Linux filesystem with non UTF-8 file names (eg latin1) then you can +use the convmv tool to convert the filesystem to UTF-8. This tool is +available in most distributions' package managers. + +If an invalid (non-UTF8) filename is read, the invalid characters will +be replaced with the unicode replacement character, '�'. rclone will +emit a debug message in this case (use -v to see), eg + + Local file system at .: Replacing invalid UTF-8 characters in "gro\xdf" + +Long paths on Windows + +Rclone handles long paths automatically, by converting all paths to long +UNC paths which allows paths up to 32,767 characters. + +This is why you will see that your paths, for instance c:\files is +converted to the UNC path \\?\c:\files in the output, and \\server\share +is converted to \\?\UNC\server\share. + +However, in rare cases this may cause problems with buggy file system +drivers like EncFS. To disable UNC conversion globally, add this to your +.rclone.conf file: + + [local] + nounc = true + +If you want to selectively disable UNC, you can add it to a separate +entry like this: + + [nounc] + type = local + nounc = true + +And use rclone like this: + +rclone copy c:\src nounc:z:\dst + +This will use UNC paths on c:\src but not on z:\dst. Of course this will +cause problems if the absolute path length of a file exceeds 258 +characters on z, so only use this option if you have to. + +Specific options + +Here are the command line options specific to local storage + +--copy-links, -L + +Normally rclone will ignore symlinks or junction points (which behave +like symlinks under Windows). + +If you supply this flag then rclone will follow the symlink and copy the +pointed to file or directory. + +This flag applies to all commands. + +For example, supposing you have a directory structure like this + + $ tree /tmp/a + /tmp/a + ├── b -> ../b + ├── expected -> ../expected + ├── one + └── two + └── three + +Then you can see the difference with and without the flag like this + + $ rclone ls /tmp/a + 6 one + 6 two/three + +and + + $ rclone -L ls /tmp/a + 4174 expected + 6 one + 6 two/three + 6 b/two + 6 b/one + +--local-no-check-updated + +Don't check to see if the files change during upload. + +Normally rclone checks the size and modification time of files as they +are being uploaded and aborts with a message which starts +can't copy - source file is being updated if the file changes during +upload. + +However on some file systems this modification time check may fail (eg +Glusterfs #2206) so this check can be disabled with this flag. + +--local-no-unicode-normalization + +This flag is deprecated now. Rclone no longer normalizes unicode file +names, but it compares them with unicode normalization in the sync +routine instead. + +--one-file-system, -x + +This tells rclone to stay in the filesystem specified by the root and +not to recurse into different file systems. + +For example if you have a directory hierarchy like this + + root + ├── disk1 - disk1 mounted on the root + │   └── file3 - stored on disk1 + ├── disk2 - disk2 mounted on the root + │   └── file4 - stored on disk12 + ├── file1 - stored on the root disk + └── file2 - stored on the root disk + +Using rclone --one-file-system copy root remote: will only copy file1 +and file2. Eg + + $ rclone -q --one-file-system ls root + 0 file1 + 0 file2 + + $ rclone -q ls root + 0 disk1/file3 + 0 disk2/file4 + 0 file1 + 0 file2 + +NB Rclone (like most unix tools such as du, rsync and tar) treats a bind +mount to the same device as being on the same filesystem. + +NB This flag is only available on Unix based systems. On systems where +it isn't supported (eg Windows) it will not appear as an valid flag. + +--skip-links + +This flag disables warning messages on skipped symlinks or junction +points, as you explicitly acknowledge that they should be skipped. + + + +CHANGELOG + + +v1.43.1 - 2018-09-07 + +Point release to fix hubic and azureblob backends. + +- Bug Fixes + - ncdu: Return error instead of log.Fatal in Show (Fabian Möller) + - cmd: Fix crash with --progress and --stats 0 (Nick Craig-Wood) + - docs: Tidy website display (Anagh Kumar Baranwal) +- Azure Blob: + - Fix multi-part uploads. (sandeepkru) +- Hubic + - Fix uploads (Nick Craig-Wood) + - Retry auth fetching if it fails to make hubic more reliable + (Nick Craig-Wood) + + +v1.43 - 2018-09-01 + +- New backends + - Jottacloud (Sebastian Bünger) +- New commands + - copyurl: copies a URL to a remote (Denis) +- New Features + - Reworked config for backends (Nick Craig-Wood) + - All backend config can now be supplied by command line, env + var or config file + - Advanced section in the config wizard for the optional items + - A large step towards rclone backends being usable in other + go software + - Allow on the fly remotes with :backend: syntax + - Stats revamp + - Add --progress/-P flag to show interactive progress (Nick + Craig-Wood) + - Show the total progress of the sync in the stats (Nick + Craig-Wood) + - Add --stats-one-line flag for single line stats (Nick + Craig-Wood) + - Added weekday schedule into --bwlimit (Mateusz) + - lsjson: Add option to show the original object IDs (Fabian + Möller) + - serve webdav: Make Content-Type without reading the file and add + --etag-hash (Nick Craig-Wood) + - build + - Build macOS with native compiler (Nick Craig-Wood) + - Update to use go1.11 for the build (Nick Craig-Wood) + - rc + - Added core/stats to return the stats (reddi1) + - version --check: Prints the current release and beta versions + (Nick Craig-Wood) +- Bug Fixes + - accounting + - Fix time to completion estimates (Nick Craig-Wood) + - Fix moving average speed for file stats (Nick Craig-Wood) + - config: Fix error reading password from piped input (Nick + Craig-Wood) + - move: Fix --delete-empty-src-dirs flag to delete all empty dirs + on move (ishuah) +- Mount + - Implement --daemon-timeout flag for OSXFUSE (Nick Craig-Wood) + - Fix mount --daemon not working with encrypted config (Alex Chen) + - Clip the number of blocks to 2^32-1 on macOS - fixes borg backup + (Nick Craig-Wood) +- VFS + - Enable vfs-read-chunk-size by default (Fabian Möller) + - Add the vfs/refresh rc command (Fabian Möller) + - Add non recursive mode to vfs/refresh rc command (Fabian Möller) + - Try to seek buffer on read only files (Fabian Möller) +- Local + - Fix crash when deprecated --local-no-unicode-normalization is + supplied (Nick Craig-Wood) + - Fix mkdir error when trying to copy files to the root of a drive + on windows (Nick Craig-Wood) +- Cache + - Fix nil pointer deref when using lsjson on cached directory + (Nick Craig-Wood) + - Fix nil pointer deref for occasional crash on playback (Nick + Craig-Wood) +- Crypt + - Fix accounting when checking hashes on upload (Nick Craig-Wood) +- Amazon Cloud Drive + - Make very clear in the docs that rclone has no ACD keys (Nick + Craig-Wood) +- Azure Blob + - Add connection string and SAS URL auth (Nick Craig-Wood) + - List the container to see if it exists (Nick Craig-Wood) + - Port new Azure Blob Storage SDK (sandeepkru) + - Added blob tier, tier between Hot, Cool and Archive. + (sandeepkru) + - Remove leading / from paths (Nick Craig-Wood) +- B2 + - Support Application Keys (Nick Craig-Wood) + - Remove leading / from paths (Nick Craig-Wood) +- Box + - Fix upload of > 2GB files on 32 bit platforms (Nick Craig-Wood) + - Make --box-commit-retries flag defaulting to 100 to fix large + uploads (Nick Craig-Wood) +- Drive + - Add --drive-keep-revision-forever flag (lewapm) + - Handle gdocs when filtering file names in list (Fabian Möller) + - Support using --fast-list for large speedups (Fabian Möller) +- FTP + - Fix Put mkParentDir failed: 521 for BunnyCDN (Nick Craig-Wood) +- Google Cloud Storage + - Fix index out of range error with --fast-list (Nick Craig-Wood) +- Jottacloud + - Fix MD5 error check (Oliver Heyme) + - Handle empty time values (Martin Polden) + - Calculate missing MD5s (Oliver Heyme) + - Docs, fixes and tests for MD5 calculation (Nick Craig-Wood) + - Add optional MimeTyper interface. (Sebastian Bünger) + - Implement optional About interface (for df support). (Sebastian + Bünger) +- Mega + - Wait for events instead of arbitrary sleeping (Nick Craig-Wood) + - Add --mega-hard-delete flag (Nick Craig-Wood) + - Fix failed logins with upper case chars in email (Nick + Craig-Wood) +- Onedrive + - Shared folder support (Yoni Jah) + - Implement DirMove (Cnly) + - Fix rmdir sometimes deleting directories with contents (Nick + Craig-Wood) +- Pcloud + - Delete half uploaded files on upload error (Nick Craig-Wood) +- Qingstor + - Remove leading / from paths (Nick Craig-Wood) +- S3 + - Fix index out of range error with --fast-list (Nick Craig-Wood) + - Add --s3-force-path-style (Nick Craig-Wood) + - Add support for KMS Key ID (bsteiss) + - Remove leading / from paths (Nick Craig-Wood) +- Swift + - Add storage_policy (Ruben Vandamme) + - Make it so just storage_url or auth_token can be overidden (Nick + Craig-Wood) + - Fix server side copy bug for unusal file names (Nick Craig-Wood) + - Remove leading / from paths (Nick Craig-Wood) +- WebDAV + - Ensure we call MKCOL with a URL with a trailing / for QNAP + interop (Nick Craig-Wood) + - If root ends with / then don't check if it is a file (Nick + Craig-Wood) + - Don't accept redirects when reading metadata (Nick Craig-Wood) + - Add bearer token (Macaroon) support for dCache (Nick Craig-Wood) + - Document dCache and Macaroons (Onno Zweers) + - Sharepoint recursion with different depth (Henning) + - Attempt to remove failed uploads (Nick Craig-Wood) +- Yandex + - Fix listing/deleting files in the root (Nick Craig-Wood) + + +v1.42 - 2018-06-16 + +- New backends + - OpenDrive (Oliver Heyme, Jakub Karlicek, ncw) +- New commands + - deletefile command (Filip Bartodziej) +- New Features + - copy, move: Copy single files directly, don't use --files-from + work-around + - this makes them much more efficient + - Implement --max-transfer flag to quit transferring at a limit + - make exit code 8 for --max-transfer exceeded + - copy: copy empty source directories to destination (Ishuah + Kariuki) + - check: Add --one-way flag (Kasper Byrdal Nielsen) + - Add siginfo handler for macOS for ctrl-T stats (kubatasiemski) + - rc + - add core/gc to run a garbage collection on demand + - enable go profiling by default on the --rc port + - return error from remote on failure + - lsf + - Add --absolute flag to add a leading / onto path names + - Add --csv flag for compliant CSV output + - Add 'm' format specifier to show the MimeType + - Implement 'i' format for showing object ID + - lsjson + - Add MimeType to the output + - Add ID field to output to show Object ID + - Add --retries-sleep flag (Benjamin Joseph Dag) + - Oauth tidy up web page and error handling (Henning Surmeier) +- Bug Fixes + - Password prompt output with --log-file fixed for unix (Filip + Bartodziej) + - Calculate ModifyWindow each time on the fly to fix various + problems (Stefan Breunig) +- Mount + - Only print "File.rename error" if there actually is an error + (Stefan Breunig) + - Delay rename if file has open writers instead of failing + outright (Stefan Breunig) + - Ensure atexit gets run on interrupt + - macOS enhancements + - Make --noappledouble --noapplexattr + - Add --volname flag and remove special chars from it + - Make Get/List/Set/Remove xattr return ENOSYS for efficiency + - Make --daemon work for macOS without CGO +- VFS + - Add --vfs-read-chunk-size and --vfs-read-chunk-size-limit + (Fabian Möller) + - Fix ChangeNotify for new or changed folders (Fabian Möller) +- Local + - Fix symlink/junction point directory handling under Windows + - NB you will need to add -L to your command line to copy + files with reparse points +- Cache + - Add non cached dirs on notifications (Remus Bunduc) + - Allow root to be expired from rc (Remus Bunduc) + - Clean remaining empty folders from temp upload path (Remus + Bunduc) + - Cache lists using batch writes (Remus Bunduc) + - Use secure websockets for HTTPS Plex addresses (John Clayton) + - Reconnect plex websocket on failures (Remus Bunduc) + - Fix panic when running without plex configs (Remus Bunduc) + - Fix root folder caching (Remus Bunduc) +- Crypt + - Check the crypted hash of files when uploading for extra data + security +- Dropbox + - Make Dropbox for business folders accessible using an initial / + in the path +- Google Cloud Storage + - Low level retry all operations if necessary +- Google Drive + - Add --drive-acknowledge-abuse to download flagged files + - Add --drive-alternate-export to fix large doc export + - Don't attempt to choose Team Drives when using rclone config + create + - Fix change list polling with team drives + - Fix ChangeNotify for folders (Fabian Möller) + - Fix about (and df on a mount) for team drives +- Onedrive + - Errorhandler for onedrive for business requests (Henning + Surmeier) +- S3 + - Adjust upload concurrency with --s3-upload-concurrency + (themylogin) + - Fix --s3-chunk-size which was always using the minimum +- SFTP + - Add --ssh-path-override flag (Piotr Oleszczyk) + - Fix slow downloads for long latency connections +- Webdav + - Add workarounds for biz.mail.ru + - Ignore Reason-Phrase in status line to fix 4shared (Rodrigo) + - Better error message generation + + +v1.41 - 2018-04-28 + +- New backends + - Mega support added + - Webdav now supports SharePoint cookie authentication (hensur) +- New commands + - link: create public link to files and folders (Stefan Breunig) + - about: gets quota info from a remote (a-roussos, ncw) + - hashsum: a generic tool for any hash to produce md5sum like + output +- New Features + - lsd: Add -R flag and fix and update docs for all ls commands + - ncdu: added a "refresh" key - CTRL-L (Keith Goldfarb) + - serve restic: Add append-only mode (Steve Kriss) + - serve restic: Disallow overwriting files in append-only mode + (Alexander Neumann) + - serve restic: Print actual listener address (Matt Holt) + - size: Add --json flag (Matthew Holt) + - sync: implement --ignore-errors (Mateusz Pabian) + - dedupe: Add dedupe largest functionality (Richard Yang) + - fs: Extend SizeSuffix to include TB and PB for rclone about + - fs: add --dump goroutines and --dump openfiles for debugging + - rc: implement core/memstats to print internal memory usage info + - rc: new call rc/pid (Michael P. Dubner) +- Compile + - Drop support for go1.6 +- Release + - Fix make tarball (Chih-Hsuan Yen) +- Bug Fixes + - filter: fix --min-age and --max-age together check + - fs: limit MaxIdleConns and MaxIdleConnsPerHost in transport + - lsd,lsf: make sure all times we output are in local time + - rc: fix setting bwlimit to unlimited + - rc: take note of the --rc-addr flag too as per the docs +- Mount + - Use About to return the correct disk total/used/free (eg in df) + - Set --attr-timeout default to 1s - fixes: + - rclone using too much memory + - rclone not serving files to samba + - excessive time listing directories + - Fix df -i (upstream fix) +- VFS + - Filter files . and .. from directory listing + - Only make the VFS cache if --vfs-cache-mode > Off +- Local + - Add --local-no-check-updated to disable updated file checks + - Retry remove on Windows sharing violation error +- Cache + - Flush the memory cache after close + - Purge file data on notification + - Always forget parent dir for notifications + - Integrate with Plex websocket + - Add rc cache/stats (seuffert) + - Add info log on notification +- Box + - Fix failure reading large directories - parse file/directory + size as float +- Dropbox + - Fix crypt+obfuscate on dropbox + - Fix repeatedly uploading the same files +- FTP + - Work around strange response from box FTP server + - More workarounds for FTP servers to fix mkParentDir error + - Fix no error on listing non-existent directory +- Google Cloud Storage + - Add service_account_credentials (Matt Holt) + - Detect bucket presence by listing it - minimises permissions + needed + - Ignore zero length directory markers +- Google Drive + - Add service_account_credentials (Matt Holt) + - Fix directory move leaving a hardlinked directory behind + - Return proper google errors when Opening files + - When initialized with a filepath, optional features used + incorrect root path (Stefan Breunig) +- HTTP + - Fix sync for servers which don't return Content-Length in HEAD +- Onedrive + - Add QuickXorHash support for OneDrive for business + - Fix socket leak in multipart session upload +- S3 + - Look in S3 named profile files for credentials + - Add --s3-disable-checksum to disable checksum uploading (Chris + Redekop) + - Hierarchical configuration support (Giri Badanahatti) + - Add in config for all the supported S3 providers + - Add One Zone Infrequent Access storage class (Craig Rachel) + - Add --use-server-modtime support (Peter Baumgartner) + - Add --s3-chunk-size option to control multipart uploads + - Ignore zero length directory markers +- SFTP + - Update docs to match code, fix typos and clarify + disable_hashcheck prompt (Michael G. Noll) + - Update docs with Synology quirks + - Fail soft with a debug on hash failure +- Swift + - Add --use-server-modtime support (Peter Baumgartner) +- Webdav + - Support SharePoint cookie authentication (hensur) + - Strip leading and trailing / off root + + +v1.40 - 2018-03-19 + +- New backends + - Alias backend to create aliases for existing remote names + (Fabian Möller) +- New commands + - lsf: list for parsing purposes (Jakub Tasiemski) + - by default this is a simple non recursive list of files and + directories + - it can be configured to add more info in an easy to parse + way + - serve restic: for serving a remote as a Restic REST endpoint + - This enables restic to use any backends that rclone can + access + - Thanks Alexander Neumann for help, patches and review + - rc: enable the remote control of a running rclone + - The running rclone must be started with --rc and related + flags. + - Currently there is support for bwlimit, and flushing for + mount and cache. +- New Features + - --max-delete flag to add a delete threshold (Bjørn Erik + Pedersen) + - All backends now support RangeOption for ranged Open + - cat: Use RangeOption for limited fetches to make more + efficient + - cryptcheck: make reading of nonce more efficient with + RangeOption + - serve http/webdav/restic + - support SSL/TLS + - add --user --pass and --htpasswd for authentication + - copy/move: detect file size change during copy/move and abort + transfer (ishuah) + - cryptdecode: added option to return encrypted file names. + (ishuah) + - lsjson: add --encrypted to show encrypted name (Jakub Tasiemski) + - Add --stats-file-name-length to specify the printed file name + length for stats (Will Gunn) +- Compile + - Code base was shuffled and factored + - backends moved into a backend directory + - large packages split up + - See the CONTRIBUTING.md doc for info as to what lives where + now + - Update to using go1.10 as the default go version + - Implement daily full integration tests +- Release + - Include a source tarball and sign it and the binaries + - Sign the git tags as part of the release process + - Add .deb and .rpm packages as part of the build + - Make a beta release for all branches on the main repo (but not + pull requests) +- Bug Fixes + - config: fixes errors on non existing config by loading config + file only on first access + - config: retry saving the config after failure (Mateusz) + - sync: when using --backup-dir don't delete files if we can't set + their modtime + - this fixes odd behaviour with Dropbox and --backup-dir + - fshttp: fix idle timeouts for HTTP connections + - serve http: fix serving files with : in - fixes + - Fix --exclude-if-present to ignore directories which it doesn't + have permission for (Iakov Davydov) + - Make accounting work properly with crypt and b2 + - remove --no-traverse flag because it is obsolete +- Mount + - Add --attr-timeout flag to control attribute caching in kernel + - this now defaults to 0 which is correct but less efficient + - see the mount docs for more info + - Add --daemon flag to allow mount to run in the background + (ishuah) + - Fix: Return ENOSYS rather than EIO on attempted link + - This fixes FileZilla accessing an rclone mount served over + sftp. + - Fix setting modtime twice + - Mount tests now run on CI for Linux (mount & cmount)/Mac/Windows + - Many bugs fixed in the VFS layer - see below +- VFS + - Many fixes for --vfs-cache-mode writes and above + - Update cached copy if we know it has changed (fixes stale + data) + - Clean path names before using them in the cache + - Disable cache cleaner if --vfs-cache-poll-interval=0 + - Fill and clean the cache immediately on startup + - Fix Windows opening every file when it stats the file + - Fix applying modtime for an open Write Handle + - Fix creation of files when truncating + - Write 0 bytes when flushing unwritten handles to avoid race + conditions in FUSE + - Downgrade "poll-interval is not supported" message to Info + - Make OpenFile and friends return EINVAL if O_RDONLY and O_TRUNC +- Local + - Downgrade "invalid cross-device link: trying copy" to debug + - Make DirMove return fs.ErrorCantDirMove to allow fallback to + Copy for cross device + - Fix race conditions updating the hashes +- Cache + - Add support for polling - cache will update when remote changes + on supported backends + - Reduce log level for Plex api + - Fix dir cache issue + - Implement --cache-db-wait-time flag + - Improve efficiency with RangeOption and RangeSeek + - Fix dirmove with temp fs enabled + - Notify vfs when using temp fs + - Offline uploading + - Remote control support for path flushing +- Amazon cloud drive + - Rclone no longer has any working keys - disable integration + tests + - Implement DirChangeNotify to notify cache/vfs/mount of changes +- Azureblob + - Don't check for bucket/container presense if listing was OK + - this makes rclone do one less request per invocation + - Improve accounting for chunked uploads +- Backblaze B2 + - Don't check for bucket/container presense if listing was OK + - this makes rclone do one less request per invocation +- Box + - Improve accounting for chunked uploads +- Dropbox + - Fix custom oauth client parameters +- Google Cloud Storage + - Don't check for bucket/container presense if listing was OK + - this makes rclone do one less request per invocation +- Google Drive + - Migrate to api v3 (Fabian Möller) + - Add scope configuration and root folder selection + - Add --drive-impersonate for service accounts + - thanks to everyone who tested, explored and contributed docs + - Add --drive-use-created-date to use created date as modified + date (nbuchanan) + - Request the export formats only when required + - This makes rclone quicker when there are no google docs + - Fix finding paths with latin1 chars (a workaround for a drive + bug) + - Fix copying of a single Google doc file + - Fix --drive-auth-owner-only to look in all directories +- HTTP + - Fix handling of directories with & in +- Onedrive + - Removed upload cutoff and always do session uploads + - this stops the creation of multiple versions on business + onedrive + - Overwrite object size value with real size when reading file. + (Victor) + - this fixes oddities when onedrive misreports the size of + images +- Pcloud + - Remove unused chunked upload flag and code +- Qingstor + - Don't check for bucket/container presense if listing was OK + - this makes rclone do one less request per invocation +- S3 + - Support hashes for multipart files (Chris Redekop) + - Initial support for IBM COS (S3) (Giri Badanahatti) + - Update docs to discourage use of v2 auth with CEPH and others + - Don't check for bucket/container presense if listing was OK + - this makes rclone do one less request per invocation + - Fix server side copy and set modtime on files with + in +- SFTP + - Add option to disable remote hash check command execution (Jon + Fautley) + - Add --sftp-ask-password flag to prompt for password when needed + (Leo R. Lundgren) + - Add set_modtime configuration option + - Fix following of symlinks + - Fix reading config file outside of Fs setup + - Fix reading $USER in username fallback not $HOME + - Fix running under crontab - Use correct OS way of reading + username +- Swift + - Fix refresh of authentication token + - in v1.39 a bug was introduced which ignored new tokens - + this fixes it + - Fix extra HEAD transaction when uploading a new file + - Don't check for bucket/container presense if listing was OK + - this makes rclone do one less request per invocation +- Webdav + - Add new time formats to support mydrive.ch and others + + +v1.39 - 2017-12-23 + +- New backends + - WebDAV + - tested with nextcloud, owncloud, put.io and others! + - Pcloud + - cache - wraps a cache around other backends (Remus Bunduc) + - useful in combination with mount + - NB this feature is in beta so use with care +- New commands + - serve command with subcommands: + - serve webdav: this implements a webdav server for any rclone + remote. + - serve http: command to serve a remote over HTTP + - config: add sub commands for full config file management + - create/delete/dump/edit/file/password/providers/show/update + - touch: to create or update the timestamp of a file (Jakub + Tasiemski) +- New Features + - curl install for rclone (Filip Bartodziej) + - --stats now shows percentage, size, rate and ETA in condensed + form (Ishuah Kariuki) + - --exclude-if-present to exclude a directory if a file is present + (Iakov Davydov) + - rmdirs: add --leave-root flag (lewpam) + - move: add --delete-empty-src-dirs flag to remove dirs after move + (Ishuah Kariuki) + - Add --dump flag, introduce --dump requests, responses and remove + --dump-auth, --dump-filters + - Obscure X-Auth-Token: from headers when dumping too + - Document and implement exit codes for different failure modes + (Ishuah Kariuki) +- Compile +- Bug Fixes + - Retry lots more different types of errors to make multipart + transfers more reliable + - Save the config before asking for a token, fixes disappearing + oauth config + - Warn the user if --include and --exclude are used together + (Ernest Borowski) + - Fix duplicate files (eg on Google drive) causing spurious copies + - Allow trailing and leading whitespace for passwords (Jason Rose) + - ncdu: fix crashes on empty directories + - rcat: fix goroutine leak + - moveto/copyto: Fix to allow copying to the same name +- Mount + - --vfs-cache mode to make writes into mounts more reliable. + - this requires caching files on the disk (see --cache-dir) + - As this is a new feature, use with care + - Use sdnotify to signal systemd the mount is ready (Fabian + Möller) + - Check if directory is not empty before mounting (Ernest + Borowski) +- Local + - Add error message for cross file system moves + - Fix equality check for times +- Dropbox + - Rework multipart upload + - buffer the chunks when uploading large files so they can be + retried + - change default chunk size to 48MB now we are buffering them + in memory + - retry every error after the first chunk is done successfully + - Fix error when renaming directories +- Swift + - Fix crash on bad authentication +- Google Drive + - Add service account support (Tim Cooijmans) +- S3 + - Make it work properly with Digital Ocean Spaces (Andrew + Starr-Bochicchio) + - Fix crash if a bad listing is received + - Add support for ECS task IAM roles (David Minor) +- Backblaze B2 + - Fix multipart upload retries + - Fix --hard-delete to make it work 100% of the time +- Swift + - Allow authentication with storage URL and auth key (Giovanni + Pizzi) + - Add new fields for swift configuration to support IBM Bluemix + Swift (Pierre Carlson) + - Add OS_TENANT_ID and OS_USER_ID to config + - Allow configs with user id instead of user name + - Check if swift segments container exists before creating (John + Leach) + - Fix memory leak in swift transfers (upstream fix) +- SFTP + - Add option to enable the use of aes128-cbc cipher (Jon Fautley) +- Amazon cloud drive + - Fix download of large files failing with "Only one auth + mechanism allowed" +- crypt + - Option to encrypt directory names or leave them intact + - Implement DirChangeNotify (Fabian Möller) +- onedrive + - Add option to choose resourceURL during setup of OneDrive + Business account if more than one is available for user + + +v1.38 - 2017-09-30 + +- New backends + - Azure Blob Storage (thanks Andrei Dragomir) + - Box + - Onedrive for Business (thanks Oliver Heyme) + - QingStor from QingCloud (thanks wuyu) +- New commands + - rcat - read from standard input and stream upload + - tree - shows a nicely formatted recursive listing + - cryptdecode - decode crypted file names (thanks ishuah) + - config show - print the config file + - config file - print the config file location +- New Features + - Empty directories are deleted on sync + - dedupe - implement merging of duplicate directories + - check and cryptcheck made more consistent and use less memory + - cleanup for remaining remotes (thanks ishuah) + - --immutable for ensuring that files don't change (thanks Jacob + McNamee) + - --user-agent option (thanks Alex McGrath Kraak) + - --disable flag to disable optional features + - --bind flag for choosing the local addr on outgoing connections + - Support for zsh auto-completion (thanks bpicode) + - Stop normalizing file names but do a normalized compare in sync +- Compile + - Update to using go1.9 as the default go version + - Remove snapd build due to maintenance problems +- Bug Fixes + - Improve retriable error detection which makes multipart uploads + better + - Make check obey --ignore-size + - Fix bwlimit toggle in conjunction with schedules (thanks + cbruegg) + - config ensures newly written config is on the same mount +- Local + - Revert to copy when moving file across file system boundaries + - --skip-links to suppress symlink warnings (thanks Zhiming Wang) +- Mount + - Re-use rcat internals to support uploads from all remotes +- Dropbox + - Fix "entry doesn't belong in directory" error + - Stop using deprecated API methods +- Swift + - Fix server side copy to empty container with --fast-list +- Google Drive + - Change the default for --drive-use-trash to true +- S3 + - Set session token when using STS (thanks Girish Ramakrishnan) + - Glacier docs and error messages (thanks Jan Varho) + - Read 1000 (not 1024) items in dir listings to fix Wasabi +- Backblaze B2 + - Fix SHA1 mismatch when downloading files with no SHA1 + - Calculate missing hashes on the fly instead of spooling + - --b2-hard-delete to permanently delete (not hide) files (thanks + John Papandriopoulos) +- Hubic + - Fix creating containers - no longer have to use the default + container +- Swift + - Optionally configure from a standard set of OpenStack + environment vars + - Add endpoint_type config +- Google Cloud Storage + - Fix bucket creation to work with limited permission users +- SFTP + - Implement connection pooling for multiple ssh connections + - Limit new connections per second + - Add support for MD5 and SHA1 hashes where available (thanks + Christian Brüggemann) +- HTTP + - Fix URL encoding issues + - Fix directories with : in + - Fix panic with URL encoded content + + +v1.37 - 2017-07-22 + +- New backends + - FTP - thanks to Antonio Messina + - HTTP - thanks to Vasiliy Tolstov +- New commands + - rclone ncdu - for exploring a remote with a text based user + interface. + - rclone lsjson - for listing with a machine readable output + - rclone dbhashsum - to show Dropbox style hashes of files (local + or Dropbox) +- New Features + - Implement --fast-list flag + - This allows remotes to list recursively if they can + - This uses less transactions (important if you pay for them) + - This may or may not be quicker + - This will use more memory as it has to hold the listing in + memory + - --old-sync-method deprecated - the remaining uses are + covered by --fast-list + - This involved a major re-write of all the listing code + - Add --tpslimit and --tpslimit-burst to limit transactions per + second + - this is useful in conjuction with rclone mount to limit + external apps + - Add --stats-log-level so can see --stats without -v + - Print password prompts to stderr - Hraban Luyat + - Warn about duplicate files when syncing + - Oauth improvements + - allow auth_url and token_url to be set in the config file + - Print redirection URI if using own credentials. + - Don't Mkdir at the start of sync to save transactions +- Compile + - Update build to go1.8.3 + - Require go1.6 for building rclone + - Compile 386 builds with "GO386=387" for maximum compatibility +- Bug Fixes + - Fix menu selection when no remotes + - Config saving reworked to not kill the file if disk gets full + - Don't delete remote if name does not change while renaming + - moveto, copyto: report transfers and checks as per move and copy +- Local + - Add --local-no-unicode-normalization flag - Bob Potter +- Mount + - Now supported on Windows using cgofuse and WinFsp - thanks to + Bill Zissimopoulos for much help + - Compare checksums on upload/download via FUSE + - Unmount when program ends with SIGINT (Ctrl+C) or SIGTERM - + Jérôme Vizcaino + - On read only open of file, make open pending until first read + - Make --read-only reject modify operations + - Implement ModTime via FUSE for remotes that support it + - Allow modTime to be changed even before all writers are closed + - Fix panic on renames + - Fix hang on errored upload +- Crypt + - Report the name:root as specified by the user + - Add an "obfuscate" option for filename encryption - Stephen + Harris +- Amazon Drive + - Fix initialization order for token renewer + - Remove revoked credentials, allow oauth proxy config and update + docs +- B2 + - Reduce minimum chunk size to 5MB +- Drive + - Add team drive support + - Reduce bandwidth by adding fields for partial responses - Martin + Kristensen + - Implement --drive-shared-with-me flag to view shared with me + files - Danny Tsai + - Add --drive-trashed-only to read only the files in the trash + - Remove obsolete --drive-full-list + - Add missing seek to start on retries of chunked uploads + - Fix stats accounting for upload + - Convert / in names to a unicode equivalent (／) + - Poll for Google Drive changes when mounted +- OneDrive + - Fix the uploading of files with spaces + - Fix initialization order for token renewer + - Display speeds accurately when uploading - Yoni Jah + - Swap to using http://localhost:53682/ as redirect URL - Michael + Ledin + - Retry on token expired error, reset upload body on retry - Yoni + Jah +- Google Cloud Storage + - Add ability to specify location and storage class via config and + command line - thanks gdm85 + - Create container if necessary on server side copy + - Increase directory listing chunk to 1000 to increase performance + - Obtain a refresh token for GCS - Steven Lu +- Yandex + - Fix the name reported in log messages (was empty) + - Correct error return for listing empty directory +- Dropbox + - Rewritten to use the v2 API + - Now supports ModTime + - Can only set by uploading the file again + - If you uploaded with an old rclone, rclone may upload + everything again + - Use --size-only or --checksum to avoid this + - Now supports the Dropbox content hashing scheme + - Now supports low level retries +- S3 + - Work around eventual consistency in bucket creation + - Create container if necessary on server side copy + - Add us-east-2 (Ohio) and eu-west-2 (London) S3 regions - Zahiar + Ahmed +- Swift, Hubic + - Fix zero length directory markers showing in the subdirectory + listing + - this caused lots of duplicate transfers + - Fix paged directory listings + - this caused duplicate directory errors + - Create container if necessary on server side copy + - Increase directory listing chunk to 1000 to increase performance + - Make sensible error if the user forgets the container +- SFTP + - Add support for using ssh key files + - Fix under Windows + - Fix ssh agent on Windows + - Adapt to latest version of library - Igor Kharin + + +v1.36 - 2017-03-18 + +- New Features + - SFTP remote (Jack Schmidt) + - Re-implement sync routine to work a directory at a time reducing + memory usage + - Logging revamped to be more inline with rsync - now much + quieter * -v only shows transfers * -vv is for full debug * + --syslog to log to syslog on capable platforms + - Implement --backup-dir and --suffix + - Implement --track-renames (initial implementation by Bjørn Erik + Pedersen) + - Add time-based bandwidth limits (Lukas Loesche) + - rclone cryptcheck: checks integrity of crypt remotes + - Allow all config file variables and options to be set from + environment variables + - Add --buffer-size parameter to control buffer size for copy + - Make --delete-after the default + - Add --ignore-checksum flag (fixed by Hisham Zarka) + - rclone check: Add --download flag to check all the data, not + just hashes + - rclone cat: add --head, --tail, --offset, --count and --discard + - rclone config: when choosing from a list, allow the value to be + entered too + - rclone config: allow rename and copy of remotes + - rclone obscure: for generating encrypted passwords for rclone's + config (T.C. Ferguson) + - Comply with XDG Base Directory specification (Dario Giovannetti) + - this moves the default location of the config file in a + backwards compatible way + - Release changes + - Ubuntu snap support (Dedsec1) + - Compile with go 1.8 + - MIPS/Linux big and little endian support +- Bug Fixes + - Fix copyto copying things to the wrong place if the destination + dir didn't exist + - Fix parsing of remotes in moveto and copyto + - Fix --delete-before deleting files on copy + - Fix --files-from with an empty file copying everything + - Fix sync: don't update mod times if --dry-run set + - Fix MimeType propagation + - Fix filters to add ** rules to directory rules +- Local + - Implement -L, --copy-links flag to allow rclone to follow + symlinks + - Open files in write only mode so rclone can write to an rclone + mount + - Fix unnormalised unicode causing problems reading directories + - Fix interaction between -x flag and --max-depth +- Mount + - Implement proper directory handling (mkdir, rmdir, renaming) + - Make include and exclude filters apply to mount + - Implement read and write async buffers - control with + --buffer-size + - Fix fsync on for directories + - Fix retry on network failure when reading off crypt +- Crypt + - Add --crypt-show-mapping to show encrypted file mapping + - Fix crypt writer getting stuck in a loop + - IMPORTANT this bug had the potential to cause data + corruption when + - reading data from a network based remote and + - writing to a crypt on Google Drive + - Use the cryptcheck command to validate your data if you are + concerned + - If syncing two crypt remotes, sync the unencrypted remote +- Amazon Drive + - Fix panics on Move (rename) + - Fix panic on token expiry +- B2 + - Fix inconsistent listings and rclone check + - Fix uploading empty files with go1.8 + - Constrain memory usage when doing multipart uploads + - Fix upload url not being refreshed properly +- Drive + - Fix Rmdir on directories with trashed files + - Fix "Ignoring unknown object" when downloading + - Add --drive-list-chunk + - Add --drive-skip-gdocs (Károly Oláh) +- OneDrive + - Implement Move + - Fix Copy + - Fix overwrite detection in Copy + - Fix waitForJob to parse errors correctly + - Use token renewer to stop auth errors on long uploads + - Fix uploading empty files with go1.8 +- Google Cloud Storage + - Fix depth 1 directory listings +- Yandex + - Fix single level directory listing +- Dropbox + - Normalise the case for single level directory listings + - Fix depth 1 listing +- S3 + - Added ca-central-1 region (Jon Yergatian) + + +v1.35 - 2017-01-02 + +- New Features + - moveto and copyto commands for choosing a destination name on + copy/move + - rmdirs command to recursively delete empty directories + - Allow repeated --include/--exclude/--filter options + - Only show transfer stats on commands which transfer stuff + - show stats on any command using the --stats flag + - Allow overlapping directories in move when server side dir move + is supported + - Add --stats-unit option - thanks Scott McGillivray +- Bug Fixes + - Fix the config file being overwritten when two rclones are + running + - Make rclone lsd obey the filters properly + - Fix compilation on mips + - Fix not transferring files that don't differ in size + - Fix panic on nil retry/fatal error +- Mount + - Retry reads on error - should help with reliability a lot + - Report the modification times for directories from the remote + - Add bandwidth accounting and limiting (fixes --bwlimit) + - If --stats provided will show stats and which files are + transferring + - Support R/W files if truncate is set. + - Implement statfs interface so df works + - Note that write is now supported on Amazon Drive + - Report number of blocks in a file - thanks Stefan Breunig +- Crypt + - Prevent the user pointing crypt at itself + - Fix failed to authenticate decrypted block errors + - these will now return the underlying unexpected EOF instead +- Amazon Drive + - Add support for server side move and directory move - thanks + Stefan Breunig + - Fix nil pointer deref on size attribute +- B2 + - Use new prefix and delimiter parameters in directory listings + - This makes --max-depth 1 dir listings as used in mount much + faster + - Reauth the account while doing uploads too - should help with + token expiry +- Drive + - Make DirMove more efficient and complain about moving the root + - Create destination directory on Move() + + +v1.34 - 2016-11-06 + +- New Features + - Stop single file and --files-from operations iterating through + the source bucket. + - Stop removing failed upload to cloud storage remotes + - Make ContentType be preserved for cloud to cloud copies + - Add support to toggle bandwidth limits via SIGUSR2 - thanks + Marco Paganini + - rclone check shows count of hashes that couldn't be checked + - rclone listremotes command + - Support linux/arm64 build - thanks Fredrik Fornwall + - Remove Authorization: lines from --dump-headers output +- Bug Fixes + - Ignore files with control characters in the names + - Fix rclone move command + - Delete src files which already existed in dst + - Fix deletion of src file when dst file older + - Fix rclone check on crypted file systems + - Make failed uploads not count as "Transferred" + - Make sure high level retries show with -q + - Use a vendor directory with godep for repeatable builds +- rclone mount - FUSE + - Implement FUSE mount options + - --no-modtime, --debug-fuse, --read-only, --allow-non-empty, + --allow-root, --allow-other + - --default-permissions, --write-back-cache, --max-read-ahead, + --umask, --uid, --gid + - Add --dir-cache-time to control caching of directory entries + - Implement seek for files opened for read (useful for video + players) + - with -no-seek flag to disable + - Fix crash on 32 bit ARM (alignment of 64 bit counter) + - ...and many more internal fixes and improvements! +- Crypt + - Don't show encrypted password in configurator to stop confusion +- Amazon Drive + - New wait for upload option --acd-upload-wait-per-gb + - upload timeouts scale by file size and can be disabled + - Add 502 Bad Gateway to list of errors we retry + - Fix overwriting a file with a zero length file + - Fix ACD file size warning limit - thanks Felix Bünemann +- Local + - Unix: implement -x/--one-file-system to stay on a single file + system + - thanks Durval Menezes and Luiz Carlos Rumbelsperger Viana + - Windows: ignore the symlink bit on files + - Windows: Ignore directory based junction points +- B2 + - Make sure each upload has at least one upload slot - fixes + strange upload stats + - Fix uploads when using crypt + - Fix download of large files (sha1 mismatch) + - Return error when we try to create a bucket which someone else + owns + - Update B2 docs with Data usage, and Crypt section - thanks + Tomasz Mazur +- S3 + - Command line and config file support for + - Setting/overriding ACL - thanks Radek Senfeld + - Setting storage class - thanks Asko Tamm +- Drive + - Make exponential backoff work exactly as per Google + specification + - add .epub, .odp and .tsv as export formats. +- Swift + - Don't read metadata for directory marker objects + + +v1.33 - 2016-08-24 + +- New Features + - Implement encryption + - data encrypted in NACL secretbox format + - with optional file name encryption + - New commands + - rclone mount - implements FUSE mounting of remotes + (EXPERIMENTAL) + - works on Linux, FreeBSD and OS X (need testers for the + last 2!) + - rclone cat - outputs remote file or files to the terminal + - rclone genautocomplete - command to make a bash completion + script for rclone + - Editing a remote using rclone config now goes through the wizard + - Compile with go 1.7 - this fixes rclone on macOS Sierra and on + 386 processors + - Use cobra for sub commands and docs generation +- drive + - Document how to make your own client_id +- s3 + - User-configurable Amazon S3 ACL (thanks Radek Šenfeld) +- b2 + - Fix stats accounting for upload - no more jumping to 100% done + - On cleanup delete hide marker if it is the current file + - New B2 API endpoint (thanks Per Cederberg) + - Set maximum backoff to 5 Minutes +- onedrive + - Fix URL escaping in file names - eg uploading files with + in + them. +- amazon cloud drive + - Fix token expiry during large uploads + - Work around 408 REQUEST_TIMEOUT and 504 GATEWAY_TIMEOUT errors +- local + - Fix filenames with invalid UTF-8 not being uploaded + - Fix problem with some UTF-8 characters on OS X + + +v1.32 - 2016-07-13 + +- Backblaze B2 + - Fix upload of files large files not in root + + +v1.31 - 2016-07-13 + +- New Features + - Reduce memory on sync by about 50% + - Implement --no-traverse flag to stop copy traversing the + destination remote. + - This can be used to reduce memory usage down to the smallest + possible. + - Useful to copy a small number of files into a large + destination folder. + - Implement cleanup command for emptying trash / removing old + versions of files + - Currently B2 only + - Single file handling improved + - Now copied with --files-from + - Automatically sets --no-traverse when copying a single file + - Info on using installing with ansible - thanks Stefan Weichinger + - Implement --no-update-modtime flag to stop rclone fixing the + remote modified times. +- Bug Fixes + - Fix move command - stop it running for overlapping Fses - this + was causing data loss. +- Local + - Fix incomplete hashes - this was causing problems for B2. +- Amazon Drive + - Rename Amazon Cloud Drive to Amazon Drive - no changes to config + file needed. +- Swift + - Add support for non-default project domain - thanks Antonio + Messina. +- S3 + - Add instructions on how to use rclone with minio. + - Add ap-northeast-2 (Seoul) and ap-south-1 (Mumbai) regions. + - Skip setting the modified time for objects > 5GB as it isn't + possible. +- Backblaze B2 + - Add --b2-versions flag so old versions can be listed and + retreived. + - Treat 403 errors (eg cap exceeded) as fatal. + - Implement cleanup command for deleting old file versions. + - Make error handling compliant with B2 integrations notes. + - Fix handling of token expiry. + - Implement --b2-test-mode to set X-Bz-Test-Mode header. + - Set cutoff for chunked upload to 200MB as per B2 guidelines. + - Make upload multi-threaded. +- Dropbox + - Don't retry 461 errors. + + +v1.30 - 2016-06-18 + +- New Features + - Directory listing code reworked for more features and better + error reporting (thanks to Klaus Post for help). This enables + - Directory include filtering for efficiency + - --max-depth parameter + - Better error reporting + - More to come + - Retry more errors + - Add --ignore-size flag - for uploading images to onedrive + - Log -v output to stdout by default + - Display the transfer stats in more human readable form + - Make 0 size files specifiable with --max-size 0b + - Add b suffix so we can specify bytes in --bwlimit, --min-size + etc + - Use "password:" instead of "password>" prompt - thanks Klaus + Post and Leigh Klotz +- Bug Fixes + - Fix retry doing one too many retries +- Local + - Fix problems with OS X and UTF-8 characters +- Amazon Drive + - Check a file exists before uploading to help with 408 Conflict + errors + - Reauth on 401 errors - this has been causing a lot of problems + - Work around spurious 403 errors + - Restart directory listings on error +- Google Drive + - Check a file exists before uploading to help with duplicates + - Fix retry of multipart uploads +- Backblaze B2 + - Implement large file uploading +- S3 + - Add AES256 server-side encryption for - thanks Justin R. Wilson +- Google Cloud Storage + - Make sure we don't use conflicting content types on upload + - Add service account support - thanks Michal Witkowski +- Swift + - Add auth version parameter + - Add domain option for openstack (v3 auth) - thanks Fabian Ruff + + +v1.29 - 2016-04-18 + +- New Features + - Implement -I, --ignore-times for unconditional upload + - Improve dedupecommand + - Now removes identical copies without asking + - Now obeys --dry-run + - Implement --dedupe-mode for non interactive running + - --dedupe-mode interactive - interactive the default. + - --dedupe-mode skip - removes identical files then skips + anything left. + - --dedupe-mode first - removes identical files then keeps + the first one. + - --dedupe-mode newest - removes identical files then + keeps the newest one. + - --dedupe-mode oldest - removes identical files then + keeps the oldest one. + - --dedupe-mode rename - removes identical files then + renames the rest to be different. +- Bug fixes + - Make rclone check obey the --size-only flag. + - Use "application/octet-stream" if discovered mime type is + invalid. + - Fix missing "quit" option when there are no remotes. +- Google Drive + - Increase default chunk size to 8 MB - increases upload speed of + big files + - Speed up directory listings and make more reliable + - Add missing retries for Move and DirMove - increases reliability + - Preserve mime type on file update +- Backblaze B2 + - Enable mod time syncing + - This means that B2 will now check modification times + - It will upload new files to update the modification times + - (there isn't an API to just set the mod time.) + - If you want the old behaviour use --size-only. + - Update API to new version + - Fix parsing of mod time when not in metadata +- Swift/Hubic + - Don't return an MD5SUM for static large objects +- S3 + - Fix uploading files bigger than 50GB + + +v1.28 - 2016-03-01 + +- New Features + - Configuration file encryption - thanks Klaus Post + - Improve rclone config adding more help and making it easier to + understand + - Implement -u/--update so creation times can be used on all + remotes + - Implement --low-level-retries flag + - Optionally disable gzip compression on downloads with + --no-gzip-encoding +- Bug fixes + - Don't make directories if --dry-run set + - Fix and document the move command + - Fix redirecting stderr on unix-like OSes when using --log-file + - Fix delete command to wait until all finished - fixes missing + deletes. +- Backblaze B2 + - Use one upload URL per go routine fixes + more than one upload using auth token + - Add pacing, retries and reauthentication - fixes token expiry + problems + - Upload without using a temporary file from local (and remotes + which support SHA1) + - Fix reading metadata for all files when it shouldn't have been +- Drive + - Fix listing drive documents at root + - Disable copy and move for Google docs +- Swift + - Fix uploading of chunked files with non ASCII characters + - Allow setting of storage_url in the config - thanks Xavier Lucas +- S3 + - Allow IAM role and credentials from environment variables - + thanks Brian Stengaard + - Allow low privilege users to use S3 (check if directory exists + during Mkdir) - thanks Jakub Gedeon +- Amazon Drive + - Retry on more things to make directory listings more reliable + + +v1.27 - 2016-01-31 + +- New Features + - Easier headless configuration with rclone authorize + - Add support for multiple hash types - we now check SHA1 as well + as MD5 hashes. + - delete command which does obey the filters (unlike purge) + - dedupe command to deduplicate a remote. Useful with Google + Drive. + - Add --ignore-existing flag to skip all files that exist on + destination. + - Add --delete-before, --delete-during, --delete-after flags. + - Add --memprofile flag to debug memory use. + - Warn the user about files with same name but different case + - Make --include rules add their implict exclude * at the end of + the filter list + - Deprecate compiling with go1.3 +- Amazon Drive + - Fix download of files > 10 GB + - Fix directory traversal ("Next token is expired") for large + directory listings + - Remove 409 conflict from error codes we will retry - stops very + long pauses +- Backblaze B2 + - SHA1 hashes now checked by rclone core +- Drive + - Add --drive-auth-owner-only to only consider files owned by the + user - thanks Björn Harrtell + - Export Google documents +- Dropbox + - Make file exclusion error controllable with -q +- Swift + - Fix upload from unprivileged user. +- S3 + - Fix updating of mod times of files with + in. +- Local + - Add local file system option to disable UNC on Windows. + + +v1.26 - 2016-01-02 + +- New Features + - Yandex storage backend - thank you Dmitry Burdeev ("dibu") + - Implement Backblaze B2 storage backend + - Add --min-age and --max-age flags - thank you Adriano Aurélio + Meirelles + - Make ls/lsl/md5sum/size/check obey includes and excludes +- Fixes + - Fix crash in http logging + - Upload releases to github too +- Swift + - Fix sync for chunked files +- OneDrive + - Re-enable server side copy + - Don't mask HTTP error codes with JSON decode error +- S3 + - Fix corrupting Content-Type on mod time update (thanks Joseph + Spurrier) + + +v1.25 - 2015-11-14 + +- New features + - Implement Hubic storage system +- Fixes + - Fix deletion of some excluded files without --delete-excluded + - This could have deleted files unexpectedly on sync + - Always check first with --dry-run! +- Swift + - Stop SetModTime losing metadata (eg X-Object-Manifest) + - This could have caused data loss for files > 5GB in size + - Use ContentType from Object to avoid lookups in listings +- OneDrive + - disable server side copy as it seems to be broken at Microsoft + + +v1.24 - 2015-11-07 + +- New features + - Add support for Microsoft OneDrive + - Add --no-check-certificate option to disable server certificate + verification + - Add async readahead buffer for faster transfer of big files +- Fixes + - Allow spaces in remotes and check remote names for validity at + creation time + - Allow '&' and disallow ':' in Windows filenames. +- Swift + - Ignore directory marker objects where appropriate - allows + working with Hubic + - Don't delete the container if fs wasn't at root +- S3 + - Don't delete the bucket if fs wasn't at root +- Google Cloud Storage + - Don't delete the bucket if fs wasn't at root + + +v1.23 - 2015-10-03 + +- New features + - Implement rclone size for measuring remotes +- Fixes + - Fix headless config for drive and gcs + - Tell the user they should try again if the webserver method + failed + - Improve output of --dump-headers +- S3 + - Allow anonymous access to public buckets +- Swift + - Stop chunked operations logging "Failed to read info: Object Not + Found" + - Use Content-Length on uploads for extra reliability + + +v1.22 - 2015-09-28 + +- Implement rsync like include and exclude flags +- swift + - Support files > 5GB - thanks Sergey Tolmachev + + +v1.21 - 2015-09-22 + +- New features + - Display individual transfer progress + - Make lsl output times in localtime +- Fixes + - Fix allowing user to override credentials again in Drive, GCS + and ACD +- Amazon Drive + - Implement compliant pacing scheme +- Google Drive + - Make directory reads concurrent for increased speed. + + +v1.20 - 2015-09-15 + +- New features + - Amazon Drive support + - Oauth support redone - fix many bugs and improve usability + - Use "golang.org/x/oauth2" as oauth libary of choice + - Improve oauth usability for smoother initial signup + - drive, googlecloudstorage: optionally use auto config for + the oauth token + - Implement --dump-headers and --dump-bodies debug flags + - Show multiple matched commands if abbreviation too short + - Implement server side move where possible +- local + - Always use UNC paths internally on Windows - fixes a lot of bugs +- dropbox + - force use of our custom transport which makes timeouts work +- Thanks to Klaus Post for lots of help with this release + + +v1.19 - 2015-08-28 + +- New features + - Server side copies for s3/swift/drive/dropbox/gcs + - Move command - uses server side copies if it can + - Implement --retries flag - tries 3 times by default + - Build for plan9/amd64 and solaris/amd64 too +- Fixes + - Make a current version download with a fixed URL for scripting + - Ignore rmdir in limited fs rather than throwing error +- dropbox + - Increase chunk size to improve upload speeds massively + - Issue an error message when trying to upload bad file name + + +v1.18 - 2015-08-17 + +- drive + - Add --drive-use-trash flag so rclone trashes instead of deletes + - Add "Forbidden to download" message for files with no + downloadURL +- dropbox + - Remove datastore + - This was deprecated and it caused a lot of problems + - Modification times and MD5SUMs no longer stored + - Fix uploading files > 2GB +- s3 + - use official AWS SDK from github.com/aws/aws-sdk-go + - NB will most likely require you to delete and recreate remote + - enable multipart upload which enables files > 5GB + - tested with Ceph / RadosGW / S3 emulation + - many thanks to Sam Liston and Brian Haymore at the Utah Center + for High Performance Computing for a Ceph test account +- misc + - Show errors when reading the config file + - Do not print stats in quiet mode - thanks Leonid Shalupov + - Add FAQ + - Fix created directories not obeying umask + - Linux installation instructions - thanks Shimon Doodkin + + +v1.17 - 2015-06-14 + +- dropbox: fix case insensitivity issues - thanks Leonid Shalupov + + +v1.16 - 2015-06-09 + +- Fix uploading big files which was causing timeouts or panics +- Don't check md5sum after download with --size-only + + +v1.15 - 2015-06-06 + +- Add --checksum flag to only discard transfers by MD5SUM - thanks + Alex Couper +- Implement --size-only flag to sync on size not checksum & modtime +- Expand docs and remove duplicated information +- Document rclone's limitations with directories +- dropbox: update docs about case insensitivity + + +v1.14 - 2015-05-21 + +- local: fix encoding of non utf-8 file names - fixes a duplicate file + problem +- drive: docs about rate limiting +- google cloud storage: Fix compile after API change in + "google.golang.org/api/storage/v1" + + +v1.13 - 2015-05-10 + +- Revise documentation (especially sync) +- Implement --timeout and --conntimeout +- s3: ignore etags from multipart uploads which aren't md5sums + + +v1.12 - 2015-03-15 + +- drive: Use chunked upload for files above a certain size +- drive: add --drive-chunk-size and --drive-upload-cutoff parameters +- drive: switch to insert from update when a failed copy deletes the + upload +- core: Log duplicate files if they are detected + + +v1.11 - 2015-03-04 + +- swift: add region parameter +- drive: fix crash on failed to update remote mtime +- In remote paths, change native directory separators to / +- Add synchronization to ls/lsl/lsd output to stop corruptions +- Ensure all stats/log messages to go stderr +- Add --log-file flag to log everything (including panics) to file +- Make it possible to disable stats printing with --stats=0 +- Implement --bwlimit to limit data transfer bandwidth + + +v1.10 - 2015-02-12 + +- s3: list an unlimited number of items +- Fix getting stuck in the configurator + + +v1.09 - 2015-02-07 + +- windows: Stop drive letters (eg C:) getting mixed up with remotes + (eg drive:) +- local: Fix directory separators on Windows +- drive: fix rate limit exceeded errors + + +v1.08 - 2015-02-04 + +- drive: fix subdirectory listing to not list entire drive +- drive: Fix SetModTime +- dropbox: adapt code to recent library changes + + +v1.07 - 2014-12-23 + +- google cloud storage: fix memory leak + + +v1.06 - 2014-12-12 + +- Fix "Couldn't find home directory" on OSX +- swift: Add tenant parameter +- Use new location of Google API packages + + +v1.05 - 2014-08-09 + +- Improved tests and consequently lots of minor fixes +- core: Fix race detected by go race detector +- core: Fixes after running errcheck +- drive: reset root directory on Rmdir and Purge +- fs: Document that Purger returns error on empty directory, test and + fix +- google cloud storage: fix ListDir on subdirectory +- google cloud storage: re-read metadata in SetModTime +- s3: make reading metadata more reliable to work around eventual + consistency problems +- s3: strip trailing / from ListDir() +- swift: return directories without / in ListDir + + +v1.04 - 2014-07-21 + +- google cloud storage: Fix crash on Update + + +v1.03 - 2014-07-20 + +- swift, s3, dropbox: fix updated files being marked as corrupted +- Make compile with go 1.1 again + + +v1.02 - 2014-07-19 + +- Implement Dropbox remote +- Implement Google Cloud Storage remote +- Verify Md5sums and Sizes after copies +- Remove times from "ls" command - lists sizes only +- Add add "lsl" - lists times and sizes +- Add "md5sum" command + + +v1.01 - 2014-07-04 + +- drive: fix transfer of big files using up lots of memory + + +v1.00 - 2014-07-03 + +- drive: fix whole second dates + + +v0.99 - 2014-06-26 + +- Fix --dry-run not working +- Make compatible with go 1.1 + + +v0.98 - 2014-05-30 + +- s3: Treat missing Content-Length as 0 for some ceph installations +- rclonetest: add file with a space in + + +v0.97 - 2014-05-05 + +- Implement copying of single files +- s3 & swift: support paths inside containers/buckets + + +v0.96 - 2014-04-24 + +- drive: Fix multiple files of same name being created +- drive: Use o.Update and fs.Put to optimise transfers +- Add version number, -V and --version + + +v0.95 - 2014-03-28 + +- rclone.org: website, docs and graphics +- drive: fix path parsing + + +v0.94 - 2014-03-27 + +- Change remote format one last time +- GNU style flags + + +v0.93 - 2014-03-16 + +- drive: store token in config file +- cross compile other versions +- set strict permissions on config file + + +v0.92 - 2014-03-15 + +- Config fixes and --config option + + +v0.91 - 2014-03-15 + +- Make config file + + +v0.90 - 2013-06-27 + +- Project named rclone + + +v0.00 - 2012-11-18 + +- Project started + + +Bugs and Limitations + +Empty directories are left behind / not created + +With remotes that have a concept of directory, eg Local and Drive, empty +directories may be left behind, or not created when one was expected. + +This is because rclone doesn't have a concept of a directory - it only +works on objects. Most of the object storage systems can't actually +store a directory so there is nowhere for rclone to store anything about +directories. + +You can work round this to some extent with thepurge command which will +delete everything under the path, INLUDING empty directories. + +This may be fixed at some point in Issue #100 + +Directory timestamps aren't preserved + +For the same reason as the above, rclone doesn't have a concept of a +directory - it only works on objects, therefore it can't preserve the +timestamps of directories. + + +Frequently Asked Questions + +Do all cloud storage systems support all rclone commands + +Yes they do. All the rclone commands (eg sync, copy etc) will work on +all the remote storage systems. + +Can I copy the config from one machine to another + +Sure! Rclone stores all of its config in a single file. If you want to +find this file, the simplest way is to run rclone -h and look at the +help for the --config flag which will tell you where it is. + +See the remote setup docs for more info. + +How do I configure rclone on a remote / headless box with no browser? + +This has now been documented in its own remote setup page. + +Can rclone sync directly from drive to s3 + +Rclone can sync between two remote cloud storage systems just fine. + +Note that it effectively downloads the file and uploads it again, so the +node running rclone would need to have lots of bandwidth. + +The syncs would be incremental (on a file by file basis). + +Eg + + rclone sync drive:Folder s3:bucket + +Using rclone from multiple locations at the same time + +You can use rclone from multiple places at the same time if you choose +different subdirectory for the output, eg + + Server A> rclone sync /tmp/whatever remote:ServerA + Server B> rclone sync /tmp/whatever remote:ServerB + +If you sync to the same directory then you should use rclone copy +otherwise the two rclones may delete each others files, eg + + Server A> rclone copy /tmp/whatever remote:Backup + Server B> rclone copy /tmp/whatever remote:Backup + +The file names you upload from Server A and Server B should be different +in this case, otherwise some file systems (eg Drive) may make +duplicates. + +Why doesn't rclone support partial transfers / binary diffs like rsync? + +Rclone stores each file you transfer as a native object on the remote +cloud storage system. This means that you can see the files you upload +as expected using alternative access methods (eg using the Google Drive +web interface). There is a 1:1 mapping between files on your hard disk +and objects created in the cloud storage system. + +Cloud storage systems (at least none I've come across yet) don't support +partially uploading an object. You can't take an existing object, and +change some bytes in the middle of it. + +It would be possible to make a sync system which stored binary diffs +instead of whole objects like rclone does, but that would break the 1:1 +mapping of files on your hard disk to objects in the remote cloud +storage system. + +All the cloud storage systems support partial downloads of content, so +it would be possible to make partial downloads work. However to make +this work efficiently this would require storing a significant amount of +metadata, which breaks the desired 1:1 mapping of files to objects. + +Can rclone do bi-directional sync? + +No, not at present. rclone only does uni-directional sync from A -> B. +It may do in the future though since it has all the primitives - it just +requires writing the algorithm to do it. + +Can I use rclone with an HTTP proxy? + +Yes. rclone will use the environment variables HTTP_PROXY, HTTPS_PROXY +and NO_PROXY, similar to cURL and other programs. + +HTTPS_PROXY takes precedence over HTTP_PROXY for https requests. + +The environment values may be either a complete URL or a "host[:port]", +in which case the "http" scheme is assumed. + +The NO_PROXY allows you to disable the proxy for specific hosts. Hosts +must be comma separated, and can contain domains or parts. For instance +"foo.com" also matches "bar.foo.com". + +Rclone gives x509: failed to load system roots and no roots provided error + +This means that rclone can't file the SSL root certificates. Likely you +are running rclone on a NAS with a cut-down Linux OS, or possibly on +Solaris. + +Rclone (via the Go runtime) tries to load the root certificates from +these places on Linux. + + "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc. + "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL + "/etc/ssl/ca-bundle.pem", // OpenSUSE + "/etc/pki/tls/cacert.pem", // OpenELEC + +So doing something like this should fix the problem. It also sets the +time which is important for SSL to work properly. + + mkdir -p /etc/ssl/certs/ + curl -o /etc/ssl/certs/ca-certificates.crt https://raw.githubusercontent.com/bagder/ca-bundle/master/ca-bundle.crt + ntpclient -s -h pool.ntp.org + +The two environment variables SSL_CERT_FILE and SSL_CERT_DIR, mentioned +in the x509 pacakge, provide an additional way to provide the SSL root +certificates. + +Note that you may need to add the --insecure option to the curl command +line if it doesn't work without. + + curl --insecure -o /etc/ssl/certs/ca-certificates.crt https://raw.githubusercontent.com/bagder/ca-bundle/master/ca-bundle.crt + +Rclone gives Failed to load config file: function not implemented error + +Likely this means that you are running rclone on Linux version not +supported by the go runtime, ie earlier than version 2.6.23. + +See the system requirements section in the go install docs for full +details. + +All my uploaded docx/xlsx/pptx files appear as archive/zip + +This is caused by uploading these files from a Windows computer which +hasn't got the Microsoft Office suite installed. The easiest way to fix +is to install the Word viewer and the Microsoft Office Compatibility +Pack for Word, Excel, and PowerPoint 2007 and later versions' file +formats + +tcp lookup some.domain.com no such host + +This happens when rclone cannot resolve a domain. Please check that your +DNS setup is generally working, e.g. + + # both should print a long list of possible IP addresses + dig www.googleapis.com # resolve using your default DNS + dig www.googleapis.com @8.8.8.8 # resolve with Google's DNS server + +If you are using systemd-resolved (default on Arch Linux), ensure it is +at version 233 or higher. Previous releases contain a bug which causes +not all domains to be resolved properly. + +Additionally with the GODEBUG=netdns= environment variable the Go +resolver decision can be influenced. This also allows to resolve certain +issues with DNS resolution. See the name resolution section in the go +docs. + + +License + +This is free software under the terms of MIT the license (check the +COPYING file included with the source code). + + Copyright (C) 2012 by Nick Craig-Wood https://www.craig-wood.com/nick/ + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + + +Authors + +- Nick Craig-Wood nick@craig-wood.com + + +Contributors + +- Alex Couper amcouper@gmail.com +- Leonid Shalupov leonid@shalupov.com shalupov@diverse.org.ru +- Shimon Doodkin helpmepro1@gmail.com +- Colin Nicholson colin@colinn.com +- Klaus Post klauspost@gmail.com +- Sergey Tolmachev tolsi.ru@gmail.com +- Adriano Aurélio Meirelles adriano@atinge.com +- C. Bess cbess@users.noreply.github.com +- Dmitry Burdeev dibu28@gmail.com +- Joseph Spurrier github@josephspurrier.com +- Björn Harrtell bjorn@wololo.org +- Xavier Lucas xavier.lucas@corp.ovh.com +- Werner Beroux werner@beroux.com +- Brian Stengaard brian@stengaard.eu +- Jakub Gedeon jgedeon@sofi.com +- Jim Tittsler jwt@onjapan.net +- Michal Witkowski michal@improbable.io +- Fabian Ruff fabian.ruff@sap.com +- Leigh Klotz klotz@quixey.com +- Romain Lapray lapray.romain@gmail.com +- Justin R. Wilson jrw972@gmail.com +- Antonio Messina antonio.s.messina@gmail.com +- Stefan G. Weichinger office@oops.co.at +- Per Cederberg cederberg@gmail.com +- Radek Šenfeld rush@logic.cz +- Fredrik Fornwall fredrik@fornwall.net +- Asko Tamm asko@deekit.net +- xor-zz xor@gstocco.com +- Tomasz Mazur tmazur90@gmail.com +- Marco Paganini paganini@paganini.net +- Felix Bünemann buenemann@louis.info +- Durval Menezes jmrclone@durval.com +- Luiz Carlos Rumbelsperger Viana maxd13_luiz_carlos@hotmail.com +- Stefan Breunig stefan-github@yrden.de +- Alishan Ladhani ali-l@users.noreply.github.com +- 0xJAKE 0xJAKE@users.noreply.github.com +- Thibault Molleman thibaultmol@users.noreply.github.com +- Scott McGillivray scott.mcgillivray@gmail.com +- Bjørn Erik Pedersen bjorn.erik.pedersen@gmail.com +- Lukas Loesche lukas@mesosphere.io +- emyarod allllaboutyou@gmail.com +- T.C. Ferguson tcf909@gmail.com +- Brandur brandur@mutelight.org +- Dario Giovannetti dev@dariogiovannetti.net +- Károly Oláh okaresz@aol.com +- Jon Yergatian jon@macfanatic.ca +- Jack Schmidt github@mowsey.org +- Dedsec1 Dedsec1@users.noreply.github.com +- Hisham Zarka hzarka@gmail.com +- Jérôme Vizcaino jerome.vizcaino@gmail.com +- Mike Tesch mjt6129@rit.edu +- Marvin Watson marvwatson@users.noreply.github.com +- Danny Tsai danny8376@gmail.com +- Yoni Jah yonjah+git@gmail.com yonjah+github@gmail.com +- Stephen Harris github@spuddy.org +- Ihor Dvoretskyi ihor.dvoretskyi@gmail.com +- Jon Craton jncraton@gmail.com +- Hraban Luyat hraban@0brg.net +- Michael Ledin mledin89@gmail.com +- Martin Kristensen me@azgul.com +- Too Much IO toomuchio@users.noreply.github.com +- Anisse Astier anisse@astier.eu +- Zahiar Ahmed zahiar@live.com +- Igor Kharin igorkharin@gmail.com +- Bill Zissimopoulos billziss@navimatics.com +- Bob Potter bobby.potter@gmail.com +- Steven Lu tacticalazn@gmail.com +- Sjur Fredriksen sjurtf@ifi.uio.no +- Ruwbin hubus12345@gmail.com +- Fabian Möller fabianm88@gmail.com f.moeller@nynex.de +- Edward Q. Bridges github@eqbridges.com +- Vasiliy Tolstov v.tolstov@selfip.ru +- Harshavardhana harsha@minio.io +- sainaen sainaen@gmail.com +- gdm85 gdm85@users.noreply.github.com +- Yaroslav Halchenko debian@onerussian.com +- John Papandriopoulos jpap@users.noreply.github.com +- Zhiming Wang zmwangx@gmail.com +- Andy Pilate cubox@cubox.me +- Oliver Heyme olihey@googlemail.com olihey@users.noreply.github.com + de8olihe@lego.com +- wuyu wuyu@yunify.com +- Andrei Dragomir adragomi@adobe.com +- Christian Brüggemann mail@cbruegg.com +- Alex McGrath Kraak amkdude@gmail.com +- bpicode bjoern.pirnay@googlemail.com +- Daniel Jagszent daniel@jagszent.de +- Josiah White thegenius2009@gmail.com +- Ishuah Kariuki kariuki@ishuah.com ishuah91@gmail.com +- Jan Varho jan@varho.org +- Girish Ramakrishnan girish@cloudron.io +- LingMan LingMan@users.noreply.github.com +- Jacob McNamee jacobmcnamee@gmail.com +- jersou jertux@gmail.com +- thierry thierry@substantiel.fr +- Simon Leinen simon.leinen@gmail.com ubuntu@s3-test.novalocal +- Dan Dascalescu ddascalescu+github@gmail.com +- Jason Rose jason@jro.io +- Andrew Starr-Bochicchio a.starr.b@gmail.com +- John Leach john@johnleach.co.uk +- Corban Raun craun@instructure.com +- Pierre Carlson mpcarl@us.ibm.com +- Ernest Borowski er.borowski@gmail.com +- Remus Bunduc remus.bunduc@gmail.com +- Iakov Davydov iakov.davydov@unil.ch dav05.gith@myths.ru +- Jakub Tasiemski tasiemski@gmail.com +- David Minor dminor@saymedia.com +- Tim Cooijmans cooijmans.tim@gmail.com +- Laurence liuxy6@gmail.com +- Giovanni Pizzi gio.piz@gmail.com +- Filip Bartodziej filipbartodziej@gmail.com +- Jon Fautley jon@dead.li +- lewapm 32110057+lewapm@users.noreply.github.com +- Yassine Imounachen yassine256@gmail.com +- Chris Redekop chris-redekop@users.noreply.github.com + chris.redekop@gmail.com +- Jon Fautley jon@adenoid.appstal.co.uk +- Will Gunn WillGunn@users.noreply.github.com +- Lucas Bremgartner lucas@bremis.ch +- Jody Frankowski jody.frankowski@gmail.com +- Andreas Roussos arouss1980@gmail.com +- nbuchanan nbuchanan@utah.gov +- Durval Menezes rclone@durval.com +- Victor vb-github@viblo.se +- Mateusz pabian.mateusz@gmail.com +- Daniel Loader spicypixel@gmail.com +- David0rk davidork@gmail.com +- Alexander Neumann alexander@bumpern.de +- Giri Badanahatti gbadanahatti@us.ibm.com@Giris-MacBook-Pro.local +- Leo R. Lundgren leo@finalresort.org +- wolfv wolfv6@users.noreply.github.com +- Dave Pedu dave@davepedu.com +- Stefan Lindblom lindblom@spotify.com +- seuffert oliver@seuffert.biz +- gbadanahatti 37121690+gbadanahatti@users.noreply.github.com +- Keith Goldfarb barkofdelight@gmail.com +- Steve Kriss steve@heptio.com +- Chih-Hsuan Yen yan12125@gmail.com +- Alexander Neumann fd0@users.noreply.github.com +- Matt Holt mholt@users.noreply.github.com +- Eri Bastos bastos.eri@gmail.com +- Michael P. Dubner pywebmail@list.ru +- Antoine GIRARD sapk@users.noreply.github.com +- Mateusz Piotrowski mpp302@gmail.com +- Animosity022 animosity22@users.noreply.github.com +- Peter Baumgartner pete@lincolnloop.com +- Craig Rachel craig@craigrachel.com +- Michael G. Noll miguno@users.noreply.github.com +- hensur me@hensur.de +- Oliver Heyme de8olihe@lego.com +- Richard Yang richard@yenforyang.com +- Piotr Oleszczyk piotr.oleszczyk@gmail.com +- Rodrigo rodarima@gmail.com +- NoLooseEnds NoLooseEnds@users.noreply.github.com +- Jakub Karlicek jakub@karlicek.me +- John Clayton john@codemonkeylabs.com +- Kasper Byrdal Nielsen byrdal76@gmail.com +- Benjamin Joseph Dag bjdag1234@users.noreply.github.com +- themylogin themylogin@gmail.com +- Onno Zweers onno.zweers@surfsara.nl +- Jasper Lievisse Adriaanse jasper@humppa.nl +- sandeepkru sandeep.ummadi@gmail.com +- HerrH atomtigerzoo@users.noreply.github.com +- Andrew 4030760+sparkyman215@users.noreply.github.com +- dan smith XX1011@gmail.com +- Oleg Kovalov iamolegkovalov@gmail.com +- Ruben Vandamme github-com-00ff86@vandamme.email +- Cnly minecnly@gmail.com +- Andres Alvarez 1671935+kir4h@users.noreply.github.com +- reddi1 xreddi@gmail.com +- Matt Tucker matthewtckr@gmail.com +- Sebastian Bünger buengese@gmail.com +- Martin Polden mpolden@mpolden.no +- Alex Chen Cnly@users.noreply.github.com +- Denis deniskovpen@gmail.com +- bsteiss 35940619+bsteiss@users.noreply.github.com + + + +CONTACT THE RCLONE PROJECT + + +Forum + +Forum for general discussions and questions: + +- https://forum.rclone.org + + +Gitub project + +The project website is at: + +- https://github.com/ncw/rclone + +There you can file bug reports, ask for help or contribute pull +requests. + + +Google+ + +Rclone has a Google+ page which announcements are posted to + +- Google+ page for general comments + + +Twitter + +You can also follow me on twitter for rclone announcements + +- [@njcw](https://twitter.com/njcw) + + +Email + +Or if all else fails or you want to ask something private or +confidential email Nick Craig-Wood diff --git a/.rclone_repo/Makefile b/.rclone_repo/Makefile new file mode 100755 index 0000000..44604b4 --- /dev/null +++ b/.rclone_repo/Makefile @@ -0,0 +1,232 @@ +SHELL = bash +BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)) +LAST_TAG := $(shell git describe --tags --abbrev=0) +ifeq ($(BRANCH),$(LAST_TAG)) + BRANCH := master +endif +TAG_BRANCH := -$(BRANCH) +BRANCH_PATH := branch/ +ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),) + TAG_BRANCH := + BRANCH_PATH := +endif +TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH) +NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f", $$_)') +ifneq ($(TAG),$(LAST_TAG)) + TAG := $(TAG)-beta +endif +GO_VERSION := $(shell go version) +GO_FILES := $(shell go list ./... | grep -v /vendor/ ) +# Run full tests if go >= go1.11 +FULL_TESTS := $(shell go version | perl -lne 'print "go$$1.$$2" if /go(\d+)\.(\d+)/ && ($$1 > 1 || $$2 >= 11)') +BETA_PATH := $(BRANCH_PATH)$(TAG) +BETA_URL := https://beta.rclone.org/$(BETA_PATH)/ +BETA_UPLOAD_ROOT := memstore:beta-rclone-org +BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH) +# Pass in GOTAGS=xyz on the make command line to set build tags +ifdef GOTAGS +BUILDTAGS=-tags "$(GOTAGS)" +endif + +.PHONY: rclone vars version + +rclone: + touch fs/version.go + go install -v --ldflags "-s -X github.com/ncw/rclone/fs.Version=$(TAG)" $(BUILDTAGS) + cp -av `go env GOPATH`/bin/rclone . + +vars: + @echo SHELL="'$(SHELL)'" + @echo BRANCH="'$(BRANCH)'" + @echo TAG="'$(TAG)'" + @echo LAST_TAG="'$(LAST_TAG)'" + @echo NEW_TAG="'$(NEW_TAG)'" + @echo GO_VERSION="'$(GO_VERSION)'" + @echo FULL_TESTS="'$(FULL_TESTS)'" + @echo BETA_URL="'$(BETA_URL)'" + +version: + @echo '$(TAG)' + +# Full suite of integration tests +test: rclone + go install github.com/ncw/rclone/fstest/test_all + -go test -v -count 1 $(BUILDTAGS) $(GO_FILES) 2>&1 | tee test.log + -test_all github.com/ncw/rclone/fs/operations github.com/ncw/rclone/fs/sync 2>&1 | tee fs/test_all.log + @echo "Written logs in test.log and fs/test_all.log" + +# Quick test +quicktest: + RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) $(GO_FILES) +ifdef FULL_TESTS + RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race $(GO_FILES) +endif + +# Do source code quality checks +check: rclone +ifdef FULL_TESTS + go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./... + errcheck $(BUILDTAGS) ./... + find . -name \*.go | grep -v /vendor/ | xargs goimports -d | grep . ; test $$? -eq 1 + go list ./... | xargs -n1 golint | grep -E -v '(StorageUrl|CdnUrl)' ; test $$? -eq 1 +else + @echo Skipping source quality tests as version of go too old +endif + +gometalinter_install: + go get -u github.com/alecthomas/gometalinter + gometalinter --install --update + +# We aren't using gometalinter as the default linter yet because +# 1. it doesn't support build tags: https://github.com/alecthomas/gometalinter/issues/275 +# 2. can't get -printfuncs working with the vet linter +gometalinter: + gometalinter ./... + +# Get the build dependencies +build_dep: +ifdef FULL_TESTS + go get -u github.com/kisielk/errcheck + go get -u golang.org/x/tools/cmd/goimports + go get -u github.com/golang/lint/golint +endif + +# Get the release dependencies +release_dep: + go get -u github.com/goreleaser/nfpm/... + go get -u github.com/aktau/github-release + +# Update dependencies +update: + GO111MODULE=on go get -u ./... + GO111MODULE=on go tidy + GO111MODULE=on go vendor + +doc: rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs + +rclone.1: MANUAL.md + pandoc -s --from markdown --to man MANUAL.md -o rclone.1 + +MANUAL.md: bin/make_manual.py docs/content/*.md commanddocs + ./bin/make_manual.py + +MANUAL.html: MANUAL.md + pandoc -s --from markdown --to html MANUAL.md -o MANUAL.html + +MANUAL.txt: MANUAL.md + pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt + +commanddocs: rclone + rclone gendocs docs/content/commands/ + +rcdocs: rclone + bin/make_rc_docs.sh + +install: rclone + install -d ${DESTDIR}/usr/bin + install -t ${DESTDIR}/usr/bin ${GOPATH}/bin/rclone + +clean: + go clean ./... + find . -name \*~ | xargs -r rm -f + rm -rf build docs/public + rm -f rclone fs/operations/operations.test fs/sync/sync.test fs/test_all.log test.log + +website: + cd docs && hugo + +upload_website: website + rclone -v sync docs/public memstore:www-rclone-org + +tarball: + git archive -9 --format=tar.gz --prefix=rclone-$(TAG)/ -o build/rclone-$(TAG).tar.gz $(TAG) + +sign_upload: + cd build && md5sum rclone-v* | gpg --clearsign > MD5SUMS + cd build && sha1sum rclone-v* | gpg --clearsign > SHA1SUMS + cd build && sha256sum rclone-v* | gpg --clearsign > SHA256SUMS + +check_sign: + cd build && gpg --verify MD5SUMS && gpg --decrypt MD5SUMS | md5sum -c + cd build && gpg --verify SHA1SUMS && gpg --decrypt SHA1SUMS | sha1sum -c + cd build && gpg --verify SHA256SUMS && gpg --decrypt SHA256SUMS | sha256sum -c + +upload: + rclone -v copy --exclude '*current*' build/ memstore:downloads-rclone-org/$(TAG) + rclone -v copy --include '*current*' --include version.txt build/ memstore:downloads-rclone-org + +upload_github: + ./bin/upload-github $(TAG) + +cross: doc + go run bin/cross-compile.go -release current $(BUILDTAGS) $(TAG) + +beta: + go run bin/cross-compile.go $(BUILDTAGS) $(TAG) + rclone -v copy build/ memstore:pub-rclone-org/$(TAG) + @echo Beta release ready at https://pub.rclone.org/$(TAG)/ + +log_since_last_release: + git log $(LAST_TAG).. + +compile_all: +ifdef FULL_TESTS + go run bin/cross-compile.go -parallel 8 -compile-only $(BUILDTAGS) $(TAG) +else + @echo Skipping compile all as version of go too old +endif + +appveyor_upload: + rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD) +ifndef BRANCH_PATH + rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT) +endif + @echo Beta release ready at $(BETA_URL) + +BUILD_FLAGS := -exclude "^(windows|darwin)/" +ifeq ($(TRAVIS_OS_NAME),osx) + BUILD_FLAGS := -include "^darwin/" -cgo +endif + +travis_beta: +ifeq ($(TRAVIS_OS_NAME),linux) + go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz' +endif + git log $(LAST_TAG).. > /tmp/git-log.txt + go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG) + rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD) +ifndef BRANCH_PATH + rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT) +endif + @echo Beta release ready at $(BETA_URL) + +# Fetch the binary builds from travis and appveyor +fetch_binaries: + rclone -v sync $(BETA_UPLOAD) build/ + +serve: website + cd docs && hugo server -v -w + +tag: doc + @echo "Old tag is $(LAST_TAG)" + @echo "New tag is $(NEW_TAG)" + echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEW_TAG)\"\n" | gofmt > fs/version.go + echo -n "$(NEW_TAG)" > docs/layouts/partials/version.html + git tag -s -m "Version $(NEW_TAG)" $(NEW_TAG) + bin/make_changelog.py $(LAST_TAG) $(NEW_TAG) > docs/content/changelog.md.new + mv docs/content/changelog.md.new docs/content/changelog.md + @echo "Edit the new changelog in docs/content/changelog.md" + @echo "Then commit all the changes" + @echo git commit -m \"Version $(NEW_TAG)\" -a -v + @echo "And finally run make retag before make cross etc" + +retag: + git tag -f -s -m "Version $(LAST_TAG)" $(LAST_TAG) + +startdev: + echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(LAST_TAG)-DEV\"\n" | gofmt > fs/version.go + git commit -m "Start $(LAST_TAG)-DEV development" fs/version.go + +winzip: + zip -9 rclone-$(TAG).zip rclone.exe + diff --git a/.rclone_repo/README.md b/.rclone_repo/README.md new file mode 100755 index 0000000..2c6ee2e --- /dev/null +++ b/.rclone_repo/README.md @@ -0,0 +1,62 @@ +[![Logo](https://rclone.org/img/rclone-120x120.png)](https://rclone.org/) + +[Website](https://rclone.org) | +[Documentation](https://rclone.org/docs/) | +[Contributing](CONTRIBUTING.md) | +[Changelog](https://rclone.org/changelog/) | +[Installation](https://rclone.org/install/) | +[Forum](https://forum.rclone.org/) +[G+](https://google.com/+RcloneOrg) + +[![Build Status](https://travis-ci.org/ncw/rclone.svg?branch=master)](https://travis-ci.org/ncw/rclone) +[![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/ncw/rclone?branch=master&passingText=windows%20-%20ok&svg=true)](https://ci.appveyor.com/project/ncw/rclone) +[![CircleCI](https://circleci.com/gh/ncw/rclone/tree/master.svg?style=svg)](https://circleci.com/gh/ncw/rclone/tree/master) +[![GoDoc](https://godoc.org/github.com/ncw/rclone?status.svg)](https://godoc.org/github.com/ncw/rclone) + +Rclone is a command line program to sync files and directories to and from + + * Amazon Drive ([See note](https://rclone.org/amazonclouddrive/#status)) + * Amazon S3 / Dreamhost / Ceph / Minio / Wasabi + * Backblaze B2 + * Box + * Dropbox + * FTP + * Google Cloud Storage + * Google Drive + * HTTP + * Hubic + * Jottacloud + * Mega + * Microsoft Azure Blob Storage + * Microsoft OneDrive + * OpenDrive + * Openstack Swift / Rackspace cloud files / Memset Memstore / OVH / Oracle Cloud Storage + * pCloud + * QingStor + * SFTP + * Webdav / Owncloud / Nextcloud + * Yandex Disk + * The local filesystem + +Features + + * MD5/SHA1 hashes checked at all times for file integrity + * Timestamps preserved on files + * Partial syncs supported on a whole file basis + * Copy mode to just copy new/changed files + * Sync (one way) mode to make a directory identical + * Check mode to check for file hash equality + * Can sync to and from network, eg two different cloud accounts + * Optional encryption (Crypt) + * Optional FUSE mount + +See the home page for installation, usage, documentation, changelog +and configuration walkthroughs. + + * https://rclone.org/ + +License +------- + +This is free software under the terms of MIT the license (check the +COPYING file included in this package). diff --git a/.rclone_repo/RELEASE.md b/.rclone_repo/RELEASE.md new file mode 100755 index 0000000..10c890b --- /dev/null +++ b/.rclone_repo/RELEASE.md @@ -0,0 +1,33 @@ +Extra required software for making a release + * [github-release](https://github.com/aktau/github-release) for uploading packages + * pandoc for making the html and man pages + +Making a release + * git status - make sure everything is checked in + * Check travis & appveyor builds are green + * make check + * make test # see integration test server or run locally + * make tag + * edit docs/content/changelog.md + * make doc + * git status - to check for new man pages - git add them + * git commit -a -v -m "Version v1.XX" + * make retag + * git push --tags origin master + * # Wait for the appveyor and travis builds to complete then... + * make fetch_binaries + * make tarball + * make sign_upload + * make check_sign + * make upload + * make upload_website + * make upload_github + * make startdev + * # announce with forum post, twitter post, G+ post + +Early in the next release cycle update the vendored dependencies + * Review any pinned packages in go.mod and remove if possible + * make update + * git status + * git add new files + * git commit -a -v diff --git a/.rclone_repo/backend/alias/alias.go b/.rclone_repo/backend/alias/alias.go new file mode 100755 index 0000000..3fcf6dc --- /dev/null +++ b/.rclone_repo/backend/alias/alias.go @@ -0,0 +1,59 @@ +package alias + +import ( + "errors" + "path" + "path/filepath" + "strings" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" +) + +// Register with Fs +func init() { + fsi := &fs.RegInfo{ + Name: "alias", + Description: "Alias for a existing remote", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "remote", + Help: "Remote or path to alias.\nCan be \"myremote:path/to/dir\", \"myremote:bucket\", \"myremote:\" or \"/local/path\".", + Required: true, + }}, + } + fs.Register(fsi) +} + +// Options defines the configuration for this backend +type Options struct { + Remote string `config:"remote"` +} + +// NewFs contstructs an Fs from the path. +// +// The returned Fs is the actual Fs, referenced by remote in the config +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + if opt.Remote == "" { + return nil, errors.New("alias can't point to an empty remote - check the value of the remote setting") + } + if strings.HasPrefix(opt.Remote, name+":") { + return nil, errors.New("can't point alias remote at itself - check the value of the remote setting") + } + _, configName, fsPath, err := fs.ParseRemote(opt.Remote) + if err != nil { + return nil, err + } + root = path.Join(fsPath, filepath.ToSlash(root)) + if configName == "local" { + return fs.NewFs(root) + } + return fs.NewFs(configName + ":" + root) +} diff --git a/.rclone_repo/backend/alias/alias_internal_test.go b/.rclone_repo/backend/alias/alias_internal_test.go new file mode 100755 index 0000000..a79769a --- /dev/null +++ b/.rclone_repo/backend/alias/alias_internal_test.go @@ -0,0 +1,104 @@ +package alias + +import ( + "fmt" + "path" + "path/filepath" + "sort" + "testing" + + _ "github.com/ncw/rclone/backend/local" // pull in test backend + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/stretchr/testify/require" +) + +var ( + remoteName = "TestAlias" +) + +func prepare(t *testing.T, root string) { + config.LoadConfig() + + // Configure the remote + config.FileSet(remoteName, "type", "alias") + config.FileSet(remoteName, "remote", root) +} + +func TestNewFS(t *testing.T) { + type testEntry struct { + remote string + size int64 + isDir bool + } + for testi, test := range []struct { + remoteRoot string + fsRoot string + fsList string + wantOK bool + entries []testEntry + }{ + {"", "", "", true, []testEntry{ + {"four", -1, true}, + {"one%.txt", 6, false}, + {"three", -1, true}, + {"two.html", 7, false}, + }}, + {"", "four", "", true, []testEntry{ + {"five", -1, true}, + {"under four.txt", 9, false}, + }}, + {"", "", "four", true, []testEntry{ + {"four/five", -1, true}, + {"four/under four.txt", 9, false}, + }}, + {"four", "..", "", true, []testEntry{ + {"four", -1, true}, + {"one%.txt", 6, false}, + {"three", -1, true}, + {"two.html", 7, false}, + }}, + {"four", "../three", "", true, []testEntry{ + {"underthree.txt", 9, false}, + }}, + } { + what := fmt.Sprintf("test %d remoteRoot=%q, fsRoot=%q, fsList=%q", testi, test.remoteRoot, test.fsRoot, test.fsList) + + remoteRoot, err := filepath.Abs(filepath.FromSlash(path.Join("test/files", test.remoteRoot))) + require.NoError(t, err, what) + prepare(t, remoteRoot) + f, err := fs.NewFs(fmt.Sprintf("%s:%s", remoteName, test.fsRoot)) + require.NoError(t, err, what) + gotEntries, err := f.List(test.fsList) + require.NoError(t, err, what) + + sort.Sort(gotEntries) + + require.Equal(t, len(test.entries), len(gotEntries), what) + for i, gotEntry := range gotEntries { + what := fmt.Sprintf("%s, entry=%d", what, i) + wantEntry := test.entries[i] + + require.Equal(t, wantEntry.remote, gotEntry.Remote(), what) + require.Equal(t, wantEntry.size, int64(gotEntry.Size()), what) + _, isDir := gotEntry.(fs.Directory) + require.Equal(t, wantEntry.isDir, isDir, what) + } + } +} + +func TestNewFSNoRemote(t *testing.T) { + prepare(t, "") + f, err := fs.NewFs(fmt.Sprintf("%s:", remoteName)) + + require.Error(t, err) + require.Nil(t, f) +} + +func TestNewFSInvalidRemote(t *testing.T) { + prepare(t, "not_existing_test_remote:") + f, err := fs.NewFs(fmt.Sprintf("%s:", remoteName)) + + require.Error(t, err) + require.Nil(t, f) +} diff --git a/.rclone_repo/backend/alias/test/files/four/five/underfive.txt b/.rclone_repo/backend/alias/test/files/four/five/underfive.txt new file mode 100755 index 0000000..4c479de --- /dev/null +++ b/.rclone_repo/backend/alias/test/files/four/five/underfive.txt @@ -0,0 +1 @@ +apple diff --git a/.rclone_repo/backend/alias/test/files/four/under four.txt b/.rclone_repo/backend/alias/test/files/four/under four.txt new file mode 100755 index 0000000..748393c --- /dev/null +++ b/.rclone_repo/backend/alias/test/files/four/under four.txt @@ -0,0 +1 @@ +beetroot diff --git a/.rclone_repo/backend/alias/test/files/one%.txt b/.rclone_repo/backend/alias/test/files/one%.txt new file mode 100755 index 0000000..ce01362 --- /dev/null +++ b/.rclone_repo/backend/alias/test/files/one%.txt @@ -0,0 +1 @@ +hello diff --git a/.rclone_repo/backend/alias/test/files/three/underthree.txt b/.rclone_repo/backend/alias/test/files/three/underthree.txt new file mode 100755 index 0000000..1031dc5 --- /dev/null +++ b/.rclone_repo/backend/alias/test/files/three/underthree.txt @@ -0,0 +1 @@ +rutabaga diff --git a/.rclone_repo/backend/alias/test/files/two.html b/.rclone_repo/backend/alias/test/files/two.html new file mode 100755 index 0000000..4bc5628 --- /dev/null +++ b/.rclone_repo/backend/alias/test/files/two.html @@ -0,0 +1 @@ +potato diff --git a/.rclone_repo/backend/all/all.go b/.rclone_repo/backend/all/all.go new file mode 100755 index 0000000..29c4bf4 --- /dev/null +++ b/.rclone_repo/backend/all/all.go @@ -0,0 +1,30 @@ +package all + +import ( + // Active file systems + _ "github.com/ncw/rclone/backend/alias" + _ "github.com/ncw/rclone/backend/amazonclouddrive" + _ "github.com/ncw/rclone/backend/azureblob" + _ "github.com/ncw/rclone/backend/b2" + _ "github.com/ncw/rclone/backend/box" + _ "github.com/ncw/rclone/backend/cache" + _ "github.com/ncw/rclone/backend/crypt" + _ "github.com/ncw/rclone/backend/drive" + _ "github.com/ncw/rclone/backend/dropbox" + _ "github.com/ncw/rclone/backend/ftp" + _ "github.com/ncw/rclone/backend/googlecloudstorage" + _ "github.com/ncw/rclone/backend/http" + _ "github.com/ncw/rclone/backend/hubic" + _ "github.com/ncw/rclone/backend/jottacloud" + _ "github.com/ncw/rclone/backend/local" + _ "github.com/ncw/rclone/backend/mega" + _ "github.com/ncw/rclone/backend/onedrive" + _ "github.com/ncw/rclone/backend/opendrive" + _ "github.com/ncw/rclone/backend/pcloud" + _ "github.com/ncw/rclone/backend/qingstor" + _ "github.com/ncw/rclone/backend/s3" + _ "github.com/ncw/rclone/backend/sftp" + _ "github.com/ncw/rclone/backend/swift" + _ "github.com/ncw/rclone/backend/webdav" + _ "github.com/ncw/rclone/backend/yandex" +) diff --git a/.rclone_repo/backend/amazonclouddrive/amazonclouddrive.go b/.rclone_repo/backend/amazonclouddrive/amazonclouddrive.go new file mode 100755 index 0000000..22201fe --- /dev/null +++ b/.rclone_repo/backend/amazonclouddrive/amazonclouddrive.go @@ -0,0 +1,1371 @@ +// Package amazonclouddrive provides an interface to the Amazon Cloud +// Drive object storage system. +package amazonclouddrive + +/* +FIXME make searching for directory in id and file in id more efficient +- use the name: search parameter - remember the escaping rules +- use Folder GetNode and GetFile + +FIXME make the default for no files and no dirs be (FILE & FOLDER) so +we ignore assets completely! +*/ + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "path" + "strings" + "time" + + "github.com/ncw/go-acd" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/dircache" + "github.com/ncw/rclone/lib/oauthutil" + "github.com/ncw/rclone/lib/pacer" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" + "golang.org/x/oauth2" +) + +const ( + folderKind = "FOLDER" + fileKind = "FILE" + statusAvailable = "AVAILABLE" + timeFormat = time.RFC3339 // 2014-03-07T22:31:12.173Z + minSleep = 20 * time.Millisecond + warnFileSize = 50000 << 20 // Display warning for files larger than this size + defaultTempLinkThreshold = fs.SizeSuffix(9 << 30) // Download files bigger than this via the tempLink +) + +// Globals +var ( + // Description of how to auth for this app + acdConfig = &oauth2.Config{ + Scopes: []string{"clouddrive:read_all", "clouddrive:write"}, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://www.amazon.com/ap/oa", + TokenURL: "https://api.amazon.com/auth/o2/token", + }, + ClientID: "", + ClientSecret: "", + RedirectURL: oauthutil.RedirectURL, + } +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "amazon cloud drive", + Prefix: "acd", + Description: "Amazon Drive", + NewFs: NewFs, + Config: func(name string, m configmap.Mapper) { + err := oauthutil.Config("amazon cloud drive", name, m, acdConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + }, + Options: []fs.Option{{ + Name: config.ConfigClientID, + Help: "Amazon Application Client ID.", + Required: true, + }, { + Name: config.ConfigClientSecret, + Help: "Amazon Application Client Secret.", + Required: true, + }, { + Name: config.ConfigAuthURL, + Help: "Auth server URL.\nLeave blank to use Amazon's.", + Advanced: true, + }, { + Name: config.ConfigTokenURL, + Help: "Token server url.\nleave blank to use Amazon's.", + Advanced: true, + }, { + Name: "checkpoint", + Help: "Checkpoint for internal polling (debug).", + Hide: fs.OptionHideBoth, + Advanced: true, + }, { + Name: "upload_wait_per_gb", + Help: "Additional time per GB to wait after a failed complete upload to see if it appears.", + Default: fs.Duration(180 * time.Second), + Advanced: true, + }, { + Name: "templink_threshold", + Help: "Files >= this size will be downloaded via their tempLink.", + Default: defaultTempLinkThreshold, + Advanced: true, + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + Checkpoint string `config:"checkpoint"` + UploadWaitPerGB fs.Duration `config:"upload_wait_per_gb"` + TempLinkThreshold fs.SizeSuffix `config:"templink_threshold"` +} + +// Fs represents a remote acd server +type Fs struct { + name string // name of this remote + features *fs.Features // optional features + opt Options // options for this Fs + c *acd.Client // the connection to the acd server + noAuthClient *http.Client // unauthenticated http client + root string // the path we are working on + dirCache *dircache.DirCache // Map of directory path to directory id + pacer *pacer.Pacer // pacer for API calls + trueRootID string // ID of true root directory + tokenRenewer *oauthutil.Renew // renew the token on expiry +} + +// Object describes a acd object +// +// Will definitely have info but maybe not meta +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + info *acd.Node // Info from the acd object if known +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("amazon drive root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// parsePath parses an acd 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 400, // Bad request (seen in "Next token is expired") + 401, // Unauthorized (seen in "Token has expired") + 408, // Request Timeout + 429, // Rate exceeded. + 500, // Get occasional 500 Internal Server Error + 502, // Bad Gateway when doing big listings + 503, // Service Unavailable + 504, // Gateway Time-out +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { + if resp != nil { + if resp.StatusCode == 401 { + f.tokenRenewer.Invalidate() + fs.Debugf(f, "401 error received - invalidating token") + return true, err + } + // Work around receiving this error sporadically on authentication + // + // HTTP code 403: "403 Forbidden", reponse body: {"message":"Authorization header requires 'Credential' parameter. Authorization header requires 'Signature' parameter. Authorization header requires 'SignedHeaders' parameter. Authorization header requires existence of either a 'X-Amz-Date' or a 'Date' header. Authorization=Bearer"} + if resp.StatusCode == 403 && strings.Contains(err.Error(), "Authorization header requires") { + fs.Debugf(f, "403 \"Authorization header requires...\" error received - retry") + return true, err + } + } + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// If query parameters contain X-Amz-Algorithm remove Authorization header +// +// This happens when ACD redirects to S3 for the download. The oauth +// transport puts an Authorization header in which we need to remove +// otherwise we get this message from AWS +// +// Only one auth mechanism allowed; only the X-Amz-Algorithm query +// parameter, Signature query string parameter or the Authorization +// header should be specified +func filterRequest(req *http.Request) { + if req.URL.Query().Get("X-Amz-Algorithm") != "" { + fs.Debugf(nil, "Removing Authorization: header after redirect to S3") + req.Header.Del("Authorization") + } +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + root = parsePath(root) + baseClient := fshttp.NewClient(fs.Config) + if do, ok := baseClient.Transport.(interface { + SetRequestFilter(f func(req *http.Request)) + }); ok { + do.SetRequestFilter(filterRequest) + } else { + fs.Debugf(name+":", "Couldn't add request filter - large file downloads will fail") + } + oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, acdConfig, baseClient) + if err != nil { + log.Fatalf("Failed to configure Amazon Drive: %v", err) + } + + c := acd.NewClient(oAuthClient) + f := &Fs{ + name: name, + root: root, + opt: *opt, + c: c, + pacer: pacer.New().SetMinSleep(minSleep).SetPacer(pacer.AmazonCloudDrivePacer), + noAuthClient: fshttp.NewClient(fs.Config), + } + f.features = (&fs.Features{ + CaseInsensitive: true, + ReadMimeType: true, + CanHaveEmptyDirectories: true, + }).Fill(f) + + // Renew the token in the background + f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { + _, err := f.getRootInfo() + return err + }) + + // Update endpoints + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + _, resp, err = f.c.Account.GetEndpoints() + return f.shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to get endpoints") + } + + // Get rootID + rootInfo, err := f.getRootInfo() + if err != nil || rootInfo.Id == nil { + return nil, errors.Wrap(err, "failed to get root") + } + f.trueRootID = *rootInfo.Id + + f.dirCache = dircache.New(root, f.trueRootID, f) + + // Find the current root + err = f.dirCache.FindRoot(false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + newF := *f + newF.dirCache = dircache.New(newRoot, f.trueRootID, &newF) + newF.root = newRoot + // Make new Fs which is the parent + err = newF.dirCache.FindRoot(false) + if err != nil { + // No root so return old f + return f, nil + } + _, err := newF.newObjectWithInfo(remote, nil) + if err != nil { + if err == fs.ErrorObjectNotFound { + // File doesn't exist so return old f + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return &newF, fs.ErrorIsFile + } + return f, nil +} + +// getRootInfo gets the root folder info +func (f *Fs) getRootInfo() (rootInfo *acd.Folder, err error) { + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + rootInfo, resp, err = f.c.Nodes.GetRoot() + return f.shouldRetry(resp, err) + }) + return rootInfo, err +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *acd.Node) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + if info != nil { + // Set info but not meta + o.info = info + } else { + err := o.readMetaData() // reads info and meta, returning an error + if err != nil { + return nil, err + } + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) { + //fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf) + folder := acd.FolderFromId(pathID, f.c.Nodes) + var resp *http.Response + var subFolder *acd.Folder + err = f.pacer.Call(func() (bool, error) { + subFolder, resp, err = folder.GetFolder(leaf) + return f.shouldRetry(resp, err) + }) + if err != nil { + if err == acd.ErrorNodeNotFound { + //fs.Debugf(f, "...Not found") + return "", false, nil + } + //fs.Debugf(f, "...Error %v", err) + return "", false, err + } + if subFolder.Status != nil && *subFolder.Status != statusAvailable { + fs.Debugf(f, "Ignoring folder %q in state %q", leaf, *subFolder.Status) + time.Sleep(1 * time.Second) // FIXME wait for problem to go away! + return "", false, nil + } + //fs.Debugf(f, "...Found(%q, %v)", *subFolder.Id, leaf) + return *subFolder.Id, true, nil +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) { + //fmt.Printf("CreateDir(%q, %q)\n", pathID, leaf) + folder := acd.FolderFromId(pathID, f.c.Nodes) + var resp *http.Response + var info *acd.Folder + err = f.pacer.Call(func() (bool, error) { + info, resp, err = folder.CreateFolder(leaf) + return f.shouldRetry(resp, err) + }) + if err != nil { + //fmt.Printf("...Error %v\n", err) + return "", err + } + //fmt.Printf("...Id %q\n", *info.Id) + return *info.Id, nil +} + +// list the objects into the function supplied +// +// If directories is set it only sends directories +// User function to process a File item from listAll +// +// Should return true to finish processing +type listAllFn func(*acd.Node) bool + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { + query := "parents:" + dirID + if directoriesOnly { + query += " AND kind:" + folderKind + } else if filesOnly { + query += " AND kind:" + fileKind + } else { + // FIXME none of these work + //query += " AND kind:(" + fileKind + " OR " + folderKind + ")" + //query += " AND (kind:" + fileKind + " OR kind:" + folderKind + ")" + } + opts := acd.NodeListOptions{ + Filters: query, + } + var nodes []*acd.Node + var out []*acd.Node + //var resp *http.Response + for { + var resp *http.Response + err = f.pacer.CallNoRetry(func() (bool, error) { + nodes, resp, err = f.c.Nodes.GetNodes(&opts) + return f.shouldRetry(resp, err) + }) + if err != nil { + return false, err + } + if nodes == nil { + break + } + for _, node := range nodes { + if node.Name != nil && node.Id != nil && node.Kind != nil && node.Status != nil { + // Ignore nodes if not AVAILABLE + if *node.Status != statusAvailable { + continue + } + // Ignore bogus nodes Amazon Drive sometimes reports + hasValidParent := false + for _, parent := range node.Parents { + if parent == dirID { + hasValidParent = true + break + } + } + if !hasValidParent { + continue + } + // Store the nodes up in case we have to retry the listing + out = append(out, node) + } + } + } + // Send the nodes now + for _, node := range out { + if fn(node) { + found = true + break + } + } + return +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + err = f.dirCache.FindRoot(false) + if err != nil { + return nil, err + } + directoryID, err := f.dirCache.FindDir(dir, false) + if err != nil { + return nil, err + } + maxTries := fs.Config.LowLevelRetries + var iErr error + for tries := 1; tries <= maxTries; tries++ { + entries = nil + _, err = f.listAll(directoryID, "", false, false, func(node *acd.Node) bool { + remote := path.Join(dir, *node.Name) + switch *node.Kind { + case folderKind: + // cache the directory ID for later lookups + f.dirCache.Put(remote, *node.Id) + when, _ := time.Parse(timeFormat, *node.ModifiedDate) // FIXME + d := fs.NewDir(remote, when).SetID(*node.Id) + entries = append(entries, d) + case fileKind: + o, err := f.newObjectWithInfo(remote, node) + if err != nil { + iErr = err + return true + } + entries = append(entries, o) + default: + // ignore ASSET etc + } + return false + }) + if iErr != nil { + return nil, iErr + } + if fserrors.IsRetryError(err) { + fs.Debugf(f, "Directory listing error for %q: %v - low level retry %d/%d", dir, err, tries, maxTries) + continue + } + if err != nil { + return nil, err + } + break + } + return entries, nil +} + +// checkUpload checks to see if an error occurred after the file was +// completely uploaded. +// +// If it was then it waits for a while to see if the file really +// exists and is the right size and returns an updated info. +// +// If the file wasn't found or was the wrong size then it returns the +// original error. +// +// This is a workaround for Amazon sometimes returning +// +// * 408 REQUEST_TIMEOUT +// * 504 GATEWAY_TIMEOUT +// * 500 Internal server error +// +// At the end of large uploads. The speculation is that the timeout +// is waiting for the sha1 hashing to complete and the file may well +// be properly uploaded. +func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, inInfo *acd.File, inErr error, uploadTime time.Duration) (fixedError bool, info *acd.File, err error) { + // Return if no error - all is well + if inErr == nil { + return false, inInfo, inErr + } + // If not one of the errors we can fix return + // if resp == nil || resp.StatusCode != 408 && resp.StatusCode != 500 && resp.StatusCode != 504 { + // return false, inInfo, inErr + // } + + // The HTTP status + httpStatus := "HTTP status UNKNOWN" + if resp != nil { + httpStatus = resp.Status + } + + // check to see if we read to the end + buf := make([]byte, 1) + n, err := in.Read(buf) + if !(n == 0 && err == io.EOF) { + fs.Debugf(src, "Upload error detected but didn't finish upload: %v (%q)", inErr, httpStatus) + return false, inInfo, inErr + } + + // Don't wait for uploads - assume they will appear later + if f.opt.UploadWaitPerGB <= 0 { + fs.Debugf(src, "Upload error detected but waiting disabled: %v (%q)", inErr, httpStatus) + return false, inInfo, inErr + } + + // Time we should wait for the upload + uploadWaitPerByte := float64(f.opt.UploadWaitPerGB) / 1024 / 1024 / 1024 + timeToWait := time.Duration(uploadWaitPerByte * float64(src.Size())) + + const sleepTime = 5 * time.Second // sleep between tries + retries := int((timeToWait + sleepTime - 1) / sleepTime) // number of retries, rounded up + + fs.Debugf(src, "Error detected after finished upload - waiting to see if object was uploaded correctly: %v (%q)", inErr, httpStatus) + remote := src.Remote() + for i := 1; i <= retries; i++ { + o, err := f.NewObject(remote) + if err == fs.ErrorObjectNotFound { + fs.Debugf(src, "Object not found - waiting (%d/%d)", i, retries) + } else if err != nil { + fs.Debugf(src, "Object returned error - waiting (%d/%d): %v", i, retries, err) + } else { + if src.Size() == o.Size() { + fs.Debugf(src, "Object found with correct size %d after waiting (%d/%d) - %v - returning with no error", src.Size(), i, retries, sleepTime*time.Duration(i-1)) + info = &acd.File{ + Node: o.(*Object).info, + } + return true, info, nil + } + fs.Debugf(src, "Object found but wrong size %d vs %d - waiting (%d/%d)", src.Size(), o.Size(), i, retries) + } + time.Sleep(sleepTime) + } + fs.Debugf(src, "Giving up waiting for object - returning original error: %v (%q)", inErr, httpStatus) + return false, inInfo, inErr +} + +// Put the object into the container +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + // Temporary Object under construction + o := &Object{ + fs: f, + remote: remote, + } + // Check if object already exists + err := o.readMetaData() + switch err { + case nil: + return o, o.Update(in, src, options...) + case fs.ErrorObjectNotFound: + // Not found so create it + default: + return nil, err + } + // If not create it + leaf, directoryID, err := f.dirCache.FindRootAndPath(remote, true) + if err != nil { + return nil, err + } + if size > warnFileSize { + fs.Logf(f, "Warning: file %q may fail because it is too big. Use --max-size=%dM to skip large files.", remote, warnFileSize>>20) + } + folder := acd.FolderFromId(directoryID, o.fs.c.Nodes) + var info *acd.File + var resp *http.Response + err = f.pacer.CallNoRetry(func() (bool, error) { + start := time.Now() + f.tokenRenewer.Start() + info, resp, err = folder.Put(in, leaf) + f.tokenRenewer.Stop() + var ok bool + ok, info, err = f.checkUpload(resp, in, src, info, err, time.Since(start)) + if ok { + return false, nil + } + return f.shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + o.info = info.Node + return o, nil +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + err := f.dirCache.FindRoot(true) + if err != nil { + return err + } + if dir != "" { + _, err = f.dirCache.FindDir(dir, true) + } + return err +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + // go test -v -run '^Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|FsMove)$' + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // create the destination directory if necessary + err := f.dirCache.FindRoot(true) + if err != nil { + return nil, err + } + srcLeaf, srcDirectoryID, err := srcObj.fs.dirCache.FindPath(srcObj.remote, false) + if err != nil { + return nil, err + } + dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(remote, true) + if err != nil { + return nil, err + } + err = f.moveNode(srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false) + if err != nil { + return nil, err + } + // Wait for directory caching so we can no longer see the old + // object and see the new object + time.Sleep(200 * time.Millisecond) // enough time 90% of the time + var ( + dstObj fs.Object + srcErr, dstErr error + ) + for i := 1; i <= fs.Config.LowLevelRetries; i++ { + _, srcErr = srcObj.fs.NewObject(srcObj.remote) // try reading the object + if srcErr != nil && srcErr != fs.ErrorObjectNotFound { + // exit if error on source + return nil, srcErr + } + dstObj, dstErr = f.NewObject(remote) + if dstErr != nil && dstErr != fs.ErrorObjectNotFound { + // exit if error on dst + return nil, dstErr + } + if srcErr == fs.ErrorObjectNotFound && dstErr == nil { + // finished if src not found and dst found + break + } + fs.Debugf(src, "Wait for directory listing to update after move %d/%d", i, fs.Config.LowLevelRetries) + time.Sleep(1 * time.Second) + } + return dstObj, dstErr +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *Fs) DirCacheFlush() { + f.dirCache.ResetRoot() +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) (err error) { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(src, "DirMove error: not same remote type") + return fs.ErrorCantDirMove + } + srcPath := path.Join(srcFs.root, srcRemote) + dstPath := path.Join(f.root, dstRemote) + + // Refuse to move to or from the root + if srcPath == "" || dstPath == "" { + fs.Debugf(src, "DirMove error: Can't move root") + return errors.New("can't move root directory") + } + + // find the root src directory + err = srcFs.dirCache.FindRoot(false) + if err != nil { + return err + } + + // find the root dst directory + if dstRemote != "" { + err = f.dirCache.FindRoot(true) + if err != nil { + return err + } + } else { + if f.dirCache.FoundRoot() { + return fs.ErrorDirExists + } + } + + // Find ID of dst parent, creating subdirs if necessary + findPath := dstRemote + if dstRemote == "" { + findPath = f.root + } + dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(findPath, true) + if err != nil { + return err + } + + // Check destination does not exist + if dstRemote != "" { + _, err = f.dirCache.FindDir(dstRemote, false) + if err == fs.ErrorDirNotFound { + // OK + } else if err != nil { + return err + } else { + return fs.ErrorDirExists + } + } + + // Find ID of src parent + findPath = srcRemote + var srcDirectoryID string + if srcRemote == "" { + srcDirectoryID, err = srcFs.dirCache.RootParentID() + } else { + _, srcDirectoryID, err = srcFs.dirCache.FindPath(findPath, false) + } + if err != nil { + return err + } + srcLeaf, _ := dircache.SplitPath(srcPath) + + // Find ID of src + srcID, err := srcFs.dirCache.FindDir(srcRemote, false) + if err != nil { + return err + } + + // FIXME make a proper node.UpdateMetadata command + srcInfo := acd.NodeFromId(srcID, f.c.Nodes) + var jsonStr string + err = srcFs.pacer.Call(func() (bool, error) { + jsonStr, err = srcInfo.GetMetadata() + return srcFs.shouldRetry(nil, err) + }) + if err != nil { + fs.Debugf(src, "DirMove error: error reading src metadata: %v", err) + return err + } + err = json.Unmarshal([]byte(jsonStr), &srcInfo) + if err != nil { + fs.Debugf(src, "DirMove error: error reading unpacking src metadata: %v", err) + return err + } + + err = f.moveNode(srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true) + if err != nil { + return err + } + + srcFs.dirCache.FlushDir(srcRemote) + return nil +} + +// purgeCheck remotes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(dir string, check bool) error { + root := path.Join(f.root, dir) + if root == "" { + return errors.New("can't purge root directory") + } + dc := f.dirCache + err := dc.FindRoot(false) + if err != nil { + return err + } + rootID, err := dc.FindDir(dir, false) + if err != nil { + return err + } + + if check { + // check directory is empty + empty := true + _, err = f.listAll(rootID, "", false, false, func(node *acd.Node) bool { + switch *node.Kind { + case folderKind: + empty = false + return true + case fileKind: + empty = false + return true + default: + fs.Debugf("Found ASSET %s", *node.Id) + } + return false + }) + if err != nil { + return err + } + if !empty { + return errors.New("directory not empty") + } + } + + node := acd.NodeFromId(rootID, f.c.Nodes) + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = node.Trash() + return f.shouldRetry(resp, err) + }) + if err != nil { + return err + } + + f.dirCache.FlushDir(dir) + if err != nil { + return err + } + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + return f.purgeCheck(dir, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +//func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { +// srcObj, ok := src.(*Object) +// if !ok { +// fs.Debugf(src, "Can't copy - not same remote type") +// return nil, fs.ErrorCantCopy +// } +// srcFs := srcObj.fs +// _, err := f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil) +// if err != nil { +// return nil, err +// } +// return f.NewObject(remote), nil +//} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + return f.purgeCheck("", false) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the Md5sum of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.MD5 { + return "", hash.ErrUnsupported + } + if o.info.ContentProperties != nil && o.info.ContentProperties.Md5 != nil { + return *o.info.ContentProperties.Md5, nil + } + return "", nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + if o.info.ContentProperties != nil && o.info.ContentProperties.Size != nil { + return int64(*o.info.ContentProperties.Size) + } + return 0 // Object is likely PENDING +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (o *Object) readMetaData() (err error) { + if o.info != nil { + return nil + } + leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(o.remote, false) + if err != nil { + if err == fs.ErrorDirNotFound { + return fs.ErrorObjectNotFound + } + return err + } + folder := acd.FolderFromId(directoryID, o.fs.c.Nodes) + var resp *http.Response + var info *acd.File + err = o.fs.pacer.Call(func() (bool, error) { + info, resp, err = folder.GetFile(leaf) + return o.fs.shouldRetry(resp, err) + }) + if err != nil { + if err == acd.ErrorNodeNotFound { + return fs.ErrorObjectNotFound + } + return err + } + o.info = info.Node + return nil +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Debugf(o, "Failed to read metadata: %v", err) + return time.Now() + } + modTime, err := time.Parse(timeFormat, *o.info.ModifiedDate) + if err != nil { + fs.Debugf(o, "Failed to read mtime from object: %v", err) + return time.Now() + } + return modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + // FIXME not implemented + return fs.ErrorCantSetModTime +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + bigObject := o.Size() >= int64(o.fs.opt.TempLinkThreshold) + if bigObject { + fs.Debugf(o, "Downloading large object via tempLink") + } + file := acd.File{Node: o.info} + var resp *http.Response + headers := fs.OpenOptionHeaders(options) + err = o.fs.pacer.Call(func() (bool, error) { + if !bigObject { + in, resp, err = file.OpenHeaders(headers) + } else { + in, resp, err = file.OpenTempURLHeaders(rest.ClientWithHeaderReset(o.fs.noAuthClient, headers), headers) + } + return o.fs.shouldRetry(resp, err) + }) + return in, err +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + file := acd.File{Node: o.info} + var info *acd.File + var resp *http.Response + var err error + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + start := time.Now() + o.fs.tokenRenewer.Start() + info, resp, err = file.Overwrite(in) + o.fs.tokenRenewer.Stop() + var ok bool + ok, info, err = o.fs.checkUpload(resp, in, src, info, err, time.Since(start)) + if ok { + return false, nil + } + return o.fs.shouldRetry(resp, err) + }) + if err != nil { + return err + } + o.info = info.Node + return nil +} + +// Remove a node +func (f *Fs) removeNode(info *acd.Node) error { + var resp *http.Response + var err error + err = f.pacer.Call(func() (bool, error) { + resp, err = info.Trash() + return f.shouldRetry(resp, err) + }) + return err +} + +// Remove an object +func (o *Object) Remove() error { + return o.fs.removeNode(o.info) +} + +// Restore a node +func (f *Fs) restoreNode(info *acd.Node) (newInfo *acd.Node, err error) { + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + newInfo, resp, err = info.Restore() + return f.shouldRetry(resp, err) + }) + return newInfo, err +} + +// Changes name of given node +func (f *Fs) renameNode(info *acd.Node, newName string) (newInfo *acd.Node, err error) { + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + newInfo, resp, err = info.Rename(newName) + return f.shouldRetry(resp, err) + }) + return newInfo, err +} + +// Replaces one parent with another, effectively moving the file. Leaves other +// parents untouched. ReplaceParent cannot be used when the file is trashed. +func (f *Fs) replaceParent(info *acd.Node, oldParentID string, newParentID string) error { + return f.pacer.Call(func() (bool, error) { + resp, err := info.ReplaceParent(oldParentID, newParentID) + return f.shouldRetry(resp, err) + }) +} + +// Adds one additional parent to object. +func (f *Fs) addParent(info *acd.Node, newParentID string) error { + return f.pacer.Call(func() (bool, error) { + resp, err := info.AddParent(newParentID) + return f.shouldRetry(resp, err) + }) +} + +// Remove given parent from object, leaving the other possible +// parents untouched. Object can end up having no parents. +func (f *Fs) removeParent(info *acd.Node, parentID string) error { + return f.pacer.Call(func() (bool, error) { + resp, err := info.RemoveParent(parentID) + return f.shouldRetry(resp, err) + }) +} + +// moveNode moves the node given from the srcLeaf,srcDirectoryID to +// the dstLeaf,dstDirectoryID +func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) { + // fs.Debugf(name, "moveNode dst(%q,%s) <- src(%q,%s)", dstLeaf, dstDirectoryID, srcLeaf, srcDirectoryID) + cantMove := fs.ErrorCantMove + if useDirErrorMsgs { + cantMove = fs.ErrorCantDirMove + } + + if len(srcInfo.Parents) > 1 && srcLeaf != dstLeaf { + fs.Debugf(name, "Move error: object is attached to multiple parents and should be renamed. This would change the name of the node in all parents.") + return cantMove + } + + if srcLeaf != dstLeaf { + // fs.Debugf(name, "renaming") + _, err = f.renameNode(srcInfo, dstLeaf) + if err != nil { + fs.Debugf(name, "Move: quick path rename failed: %v", err) + goto OnConflict + } + } + if srcDirectoryID != dstDirectoryID { + // fs.Debugf(name, "trying parent replace: %s -> %s", oldParentID, newParentID) + err = f.replaceParent(srcInfo, srcDirectoryID, dstDirectoryID) + if err != nil { + fs.Debugf(name, "Move: quick path parent replace failed: %v", err) + return err + } + } + + return nil + +OnConflict: + fs.Debugf(name, "Could not directly rename file, presumably because there was a file with the same name already. Instead, the file will now be trashed where such operations do not cause errors. It will be restored to the correct parent after. If any of the subsequent calls fails, the rename/move will be in an invalid state.") + + // fs.Debugf(name, "Trashing file") + err = f.removeNode(srcInfo) + if err != nil { + fs.Debugf(name, "Move: remove node failed: %v", err) + return err + } + // fs.Debugf(name, "Renaming file") + _, err = f.renameNode(srcInfo, dstLeaf) + if err != nil { + fs.Debugf(name, "Move: rename node failed: %v", err) + return err + } + // note: replacing parent is forbidden by API, modifying them individually is + // okay though + // fs.Debugf(name, "Adding target parent") + err = f.addParent(srcInfo, dstDirectoryID) + if err != nil { + fs.Debugf(name, "Move: addParent failed: %v", err) + return err + } + // fs.Debugf(name, "removing original parent") + err = f.removeParent(srcInfo, srcDirectoryID) + if err != nil { + fs.Debugf(name, "Move: removeParent failed: %v", err) + return err + } + // fs.Debugf(name, "Restoring") + _, err = f.restoreNode(srcInfo) + if err != nil { + fs.Debugf(name, "Move: restoreNode node failed: %v", err) + return err + } + return nil +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + if o.info.ContentProperties != nil && o.info.ContentProperties.ContentType != nil { + return *o.info.ContentProperties.ContentType + } + return "" +} + +// ChangeNotify calls the passed function with a path that has had changes. +// If the implementation uses polling, it should adhere to the given interval. +// +// Automatically restarts itself in case of unexpected behaviour of the remote. +// +// Close the returned channel to stop being notified. +func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool { + checkpoint := f.opt.Checkpoint + + quit := make(chan bool) + go func() { + for { + checkpoint = f.changeNotifyRunner(notifyFunc, checkpoint) + if err := config.SetValueAndSave(f.name, "checkpoint", checkpoint); err != nil { + fs.Debugf(f, "Unable to save checkpoint: %v", err) + } + select { + case <-quit: + return + case <-time.After(pollInterval): + } + } + }() + return quit +} + +func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), checkpoint string) string { + var err error + var resp *http.Response + var reachedEnd bool + var csCount int + var nodeCount int + + fs.Debugf(f, "Checking for changes on remote (Checkpoint %q)", checkpoint) + err = f.pacer.CallNoRetry(func() (bool, error) { + resp, err = f.c.Changes.GetChangesFunc(&acd.ChangesOptions{ + Checkpoint: checkpoint, + IncludePurged: true, + }, func(changeSet *acd.ChangeSet, err error) error { + if err != nil { + return err + } + + type entryType struct { + path string + entryType fs.EntryType + } + var pathsToClear []entryType + csCount++ + nodeCount += len(changeSet.Nodes) + if changeSet.End { + reachedEnd = true + } + if changeSet.Checkpoint != "" { + checkpoint = changeSet.Checkpoint + } + for _, node := range changeSet.Nodes { + if path, ok := f.dirCache.GetInv(*node.Id); ok { + if node.IsFile() { + pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryObject}) + } else { + pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryDirectory}) + } + continue + } + + if node.IsFile() { + // translate the parent dir of this object + if len(node.Parents) > 0 { + if path, ok := f.dirCache.GetInv(node.Parents[0]); ok { + // and append the drive file name to compute the full file name + if len(path) > 0 { + path = path + "/" + *node.Name + } else { + path = *node.Name + } + // this will now clear the actual file too + pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryObject}) + } + } else { // a true root object that is changed + pathsToClear = append(pathsToClear, entryType{path: *node.Name, entryType: fs.EntryObject}) + } + } + } + + visitedPaths := make(map[string]bool) + for _, entry := range pathsToClear { + if _, ok := visitedPaths[entry.path]; ok { + continue + } + visitedPaths[entry.path] = true + notifyFunc(entry.path, entry.entryType) + } + + return nil + }) + return false, err + }) + fs.Debugf(f, "Got %d ChangeSets with %d Nodes", csCount, nodeCount) + + if err != nil && err != io.ErrUnexpectedEOF { + fs.Debugf(f, "Failed to get Changes: %v", err) + return checkpoint + } + + if reachedEnd { + reachedEnd = false + fs.Debugf(f, "All changes were processed. Waiting for more.") + } else if checkpoint == "" { + fs.Debugf(f, "Did not get any checkpoint, something went wrong! %+v", resp) + } + return checkpoint +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + if o.info.Id == nil { + return "" + } + return *o.info.Id +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + // _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.ChangeNotifier = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = &Object{} + _ fs.IDer = &Object{} +) diff --git a/.rclone_repo/backend/amazonclouddrive/amazonclouddrive_test.go b/.rclone_repo/backend/amazonclouddrive/amazonclouddrive_test.go new file mode 100755 index 0000000..6298b93 --- /dev/null +++ b/.rclone_repo/backend/amazonclouddrive/amazonclouddrive_test.go @@ -0,0 +1,20 @@ +// Test AmazonCloudDrive filesystem interface + +// +build acd + +package amazonclouddrive_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/amazonclouddrive" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.NilObject = fs.Object((*amazonclouddrive.Object)(nil)) + fstests.RemoteName = "TestAmazonCloudDrive:" + fstests.Run(t) +} diff --git a/.rclone_repo/backend/azureblob/azureblob.go b/.rclone_repo/backend/azureblob/azureblob.go new file mode 100755 index 0000000..a6701c9 --- /dev/null +++ b/.rclone_repo/backend/azureblob/azureblob.go @@ -0,0 +1,1281 @@ +// Package azureblob provides an interface to the Microsoft Azure blob object storage system + +// +build !freebsd,!netbsd,!openbsd,!plan9,!solaris,go1.8 + +package azureblob + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + "path" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/Azure/azure-storage-blob-go/2018-03-28/azblob" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/accounting" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/fs/walk" + "github.com/ncw/rclone/lib/pacer" + "github.com/pkg/errors" +) + +const ( + minSleep = 10 * time.Millisecond + maxSleep = 10 * time.Second + decayConstant = 1 // bigger for slower decay, exponential + listChunkSize = 5000 // number of items to read at once + modTimeKey = "mtime" + timeFormatIn = time.RFC3339 + timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00" + maxTotalParts = 50000 // in multipart upload + storageDefaultBaseURL = "blob.core.windows.net" + // maxUncommittedSize = 9 << 30 // can't upload bigger than this + defaultChunkSize = 4 * 1024 * 1024 + maxChunkSize = 100 * 1024 * 1024 + defaultUploadCutoff = 256 * 1024 * 1024 + maxUploadCutoff = 256 * 1024 * 1024 + defaultAccessTier = azblob.AccessTierNone +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "azureblob", + Description: "Microsoft Azure Blob Storage", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "account", + Help: "Storage Account Name (leave blank to use connection string or SAS URL)", + }, { + Name: "key", + Help: "Storage Account Key (leave blank to use connection string or SAS URL)", + }, { + Name: "sas_url", + Help: "SAS URL for container level access only\n(leave blank if using account/key or connection string)", + }, { + Name: "endpoint", + Help: "Endpoint for the service\nLeave blank normally.", + Advanced: true, + }, { + Name: "upload_cutoff", + Help: "Cutoff for switching to chunked upload.", + Default: fs.SizeSuffix(defaultUploadCutoff), + Advanced: true, + }, { + Name: "chunk_size", + Help: "Upload chunk size. Must fit in memory.", + Default: fs.SizeSuffix(defaultChunkSize), + Advanced: true, + }, { + Name: "access_tier", + Help: "Access tier of blob, supports hot, cool and archive tiers.\nArchived blobs can be restored by setting access tier to hot or cool." + + " Leave blank if you intend to use default access tier, which is set at account level", + Advanced: true, + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + Account string `config:"account"` + Key string `config:"key"` + Endpoint string `config:"endpoint"` + SASURL string `config:"sas_url"` + UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` + AccessTier string `config:"access_tier"` +} + +// Fs represents a remote azure server +type Fs struct { + name string // name of this remote + root string // the path we are working on if any + opt Options // parsed config options + features *fs.Features // optional features + svcURL *azblob.ServiceURL // reference to serviceURL + cntURL *azblob.ContainerURL // reference to containerURL + container string // the container we are working on + containerOKMu sync.Mutex // mutex to protect container OK + containerOK bool // true if we have created the container + containerDeleted bool // true if we have deleted the container + pacer *pacer.Pacer // To pace and retry the API calls + uploadToken *pacer.TokenDispenser // control concurrency +} + +// Object describes a azure object +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + modTime time.Time // The modified time of the object if known + md5 string // MD5 hash if known + size int64 // Size of the object + mimeType string // Content-Type of the object + accessTier azblob.AccessTierType // Blob Access Tier + meta map[string]string // blob metadata +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + if f.root == "" { + return f.container + } + return f.container + "/" + f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + if f.root == "" { + return fmt.Sprintf("Azure container %s", f.container) + } + return fmt.Sprintf("Azure container %s path %s", f.container, f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Pattern to match a azure path +var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`) + +// parseParse parses a azure 'url' +func parsePath(path string) (container, directory string, err error) { + parts := matcher.FindStringSubmatch(path) + if parts == nil { + err = errors.Errorf("couldn't find container in azure path %q", path) + } else { + container, directory = parts[1], parts[2] + directory = strings.Trim(directory, "/") + } + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 401, // Unauthorized (eg "Token has expired") + 408, // Request Timeout + 429, // Rate exceeded. + 500, // Get occasional 500 Internal Server Error + 503, // Service Unavailable + 504, // Gateway Time-out +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func (f *Fs) shouldRetry(err error) (bool, error) { + // FIXME interpret special errors - more to do here + if storageErr, ok := err.(azblob.StorageError); ok { + statusCode := storageErr.Response().StatusCode + for _, e := range retryErrorCodes { + if statusCode == e { + return true, err + } + } + } + return fserrors.ShouldRetry(err), err +} + +// NewFs contstructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + if opt.UploadCutoff > maxUploadCutoff { + return nil, errors.Errorf("azure: upload cutoff (%v) must be less than or equal to %v", opt.UploadCutoff, maxUploadCutoff) + } + if opt.ChunkSize > maxChunkSize { + return nil, errors.Errorf("azure: chunk size can't be greater than %v - was %v", maxChunkSize, opt.ChunkSize) + } + container, directory, err := parsePath(root) + if err != nil { + return nil, err + } + if opt.Endpoint == "" { + opt.Endpoint = storageDefaultBaseURL + } + + if opt.AccessTier == "" { + opt.AccessTier = string(defaultAccessTier) + } else { + switch opt.AccessTier { + case string(azblob.AccessTierHot): + case string(azblob.AccessTierCool): + case string(azblob.AccessTierArchive): + // valid cases + default: + return nil, errors.Errorf("azure: Supported access tiers are %s, %s and %s", string(azblob.AccessTierHot), string(azblob.AccessTierCool), azblob.AccessTierArchive) + } + } + + var ( + u *url.URL + serviceURL azblob.ServiceURL + containerURL azblob.ContainerURL + ) + switch { + case opt.Account != "" && opt.Key != "": + credential := azblob.NewSharedKeyCredential(opt.Account, opt.Key) + u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint)) + if err != nil { + return nil, errors.Wrap(err, "failed to make azure storage url from account and endpoint") + } + pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + serviceURL = azblob.NewServiceURL(*u, pipeline) + containerURL = serviceURL.NewContainerURL(container) + case opt.SASURL != "": + u, err = url.Parse(opt.SASURL) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse SAS URL") + } + // use anonymous credentials in case of sas url + pipeline := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) + // Check if we have container level SAS or account level sas + parts := azblob.NewBlobURLParts(*u) + if parts.ContainerName != "" { + if container != "" && parts.ContainerName != container { + return nil, errors.New("Container name in SAS URL and container provided in command do not match") + } + + container = parts.ContainerName + containerURL = azblob.NewContainerURL(*u, pipeline) + } else { + serviceURL = azblob.NewServiceURL(*u, pipeline) + containerURL = serviceURL.NewContainerURL(container) + } + default: + return nil, errors.New("Need account+key or connectionString or sasURL") + } + + f := &Fs{ + name: name, + opt: *opt, + container: container, + root: directory, + svcURL: &serviceURL, + cntURL: &containerURL, + pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), + uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers), + } + f.features = (&fs.Features{ + ReadMimeType: true, + WriteMimeType: true, + BucketBased: true, + }).Fill(f) + if f.root != "" { + f.root += "/" + // Check to see if the (container,directory) is actually an existing file + oldRoot := f.root + remote := path.Base(directory) + f.root = path.Dir(directory) + if f.root == "." { + f.root = "" + } else { + f.root += "/" + } + _, err := f.NewObject(remote) + if err != nil { + if err == fs.ErrorObjectNotFound { + // File doesn't exist so return old f + f.root = oldRoot + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + return f, nil +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *azblob.BlobItem) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + if info != nil { + err := o.decodeMetaDataFromBlob(info) + if err != nil { + return nil, err + } + } else { + err := o.readMetaData() // reads info and headers, returning an error + if err != nil { + return nil, err + } + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// getBlobReference creates an empty blob reference with no metadata +func (f *Fs) getBlobReference(remote string) azblob.BlobURL { + return f.cntURL.NewBlobURL(f.root + remote) +} + +// updateMetadataWithModTime adds the modTime passed in to o.meta. +func (o *Object) updateMetadataWithModTime(modTime time.Time) { + // Make sure o.meta is not nil + if o.meta == nil { + o.meta = make(map[string]string, 1) + } + + // Set modTimeKey in it + o.meta[modTimeKey] = modTime.Format(timeFormatOut) +} + +// listFn is called from list to handle an object +type listFn func(remote string, object *azblob.BlobItem, isDirectory bool) error + +// list lists the objects into the function supplied from +// the container and root supplied +// +// dir is the starting directory, "" for root +func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error { + f.containerOKMu.Lock() + deleted := f.containerDeleted + f.containerOKMu.Unlock() + if deleted { + return fs.ErrorDirNotFound + } + root := f.root + if dir != "" { + root += dir + "/" + } + delimiter := "" + if !recurse { + delimiter = "/" + } + + options := azblob.ListBlobsSegmentOptions{ + Details: azblob.BlobListingDetails{ + Copy: false, + Metadata: true, + Snapshots: false, + UncommittedBlobs: false, + Deleted: false, + }, + Prefix: root, + MaxResults: int32(maxResults), + } + ctx := context.Background() + for marker := (azblob.Marker{}); marker.NotDone(); { + var response *azblob.ListBlobsHierarchySegmentResponse + err := f.pacer.Call(func() (bool, error) { + var err error + response, err = f.cntURL.ListBlobsHierarchySegment(ctx, marker, delimiter, options) + return f.shouldRetry(err) + }) + + if err != nil { + // Check http error code along with service code, current SDK doesn't populate service code correctly sometimes + if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) { + return fs.ErrorDirNotFound + } + return err + } + // Advance marker to next + marker = response.NextMarker + + for i := range response.Segment.BlobItems { + file := &response.Segment.BlobItems[i] + // Finish if file name no longer has prefix + // if prefix != "" && !strings.HasPrefix(file.Name, prefix) { + // return nil + // } + if !strings.HasPrefix(file.Name, f.root) { + fs.Debugf(f, "Odd name received %q", file.Name) + continue + } + remote := file.Name[len(f.root):] + // Check for directory + isDirectory := strings.HasSuffix(remote, "/") + if isDirectory { + remote = remote[:len(remote)-1] + } + // Send object + err = fn(remote, file, isDirectory) + if err != nil { + return err + } + } + // Send the subdirectories + for _, remote := range response.Segment.BlobPrefixes { + remote := strings.TrimRight(remote.Name, "/") + if !strings.HasPrefix(remote, f.root) { + fs.Debugf(f, "Odd directory name received %q", remote) + continue + } + remote = remote[len(f.root):] + // Send object + err = fn(remote, nil, true) + if err != nil { + return err + } + } + } + return nil +} + +// Convert a list item into a DirEntry +func (f *Fs) itemToDirEntry(remote string, object *azblob.BlobItem, isDirectory bool) (fs.DirEntry, error) { + if isDirectory { + d := fs.NewDir(remote, time.Time{}) + return d, nil + } + o, err := f.newObjectWithInfo(remote, object) + if err != nil { + return nil, err + } + return o, nil +} + +// mark the container as being OK +func (f *Fs) markContainerOK() { + if f.container != "" { + f.containerOKMu.Lock() + f.containerOK = true + f.containerDeleted = false + f.containerOKMu.Unlock() + } +} + +// listDir lists a single directory +func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { + err = f.list(dir, false, listChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + if entry != nil { + entries = append(entries, entry) + } + return nil + }) + if err != nil { + return nil, err + } + // container must be present if listing succeeded + f.markContainerOK() + return entries, nil +} + +// listContainers returns all the containers to out +func (f *Fs) listContainers(dir string) (entries fs.DirEntries, err error) { + if dir != "" { + return nil, fs.ErrorListBucketRequired + } + err = f.listContainersToFn(func(container *azblob.ContainerItem) error { + d := fs.NewDir(container.Name, container.Properties.LastModified) + entries = append(entries, d) + return nil + }) + if err != nil { + return nil, err + } + return entries, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + if f.container == "" { + return f.listContainers(dir) + } + return f.listDir(dir) +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + if f.container == "" { + return fs.ErrorListBucketRequired + } + list := walk.NewListRHelper(callback) + err = f.list(dir, true, listChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + return list.Add(entry) + }) + if err != nil { + return err + } + // container must be present if listing succeeded + f.markContainerOK() + return list.Flush() +} + +// listContainerFn is called from listContainersToFn to handle a container +type listContainerFn func(*azblob.ContainerItem) error + +// listContainersToFn lists the containers to the function supplied +func (f *Fs) listContainersToFn(fn listContainerFn) error { + params := azblob.ListContainersSegmentOptions{ + MaxResults: int32(listChunkSize), + } + ctx := context.Background() + for marker := (azblob.Marker{}); marker.NotDone(); { + var response *azblob.ListContainersResponse + err := f.pacer.Call(func() (bool, error) { + var err error + response, err = f.svcURL.ListContainersSegment(ctx, marker, params) + return f.shouldRetry(err) + }) + if err != nil { + return err + } + + for i := range response.ContainerItems { + err = fn(&response.ContainerItems[i]) + if err != nil { + return err + } + } + marker = response.NextMarker + } + + return nil +} + +// Put the object into the container +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + // Temporary Object under construction + fs := &Object{ + fs: f, + remote: src.Remote(), + } + return fs, fs.Update(in, src, options...) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + f.containerOKMu.Lock() + defer f.containerOKMu.Unlock() + if f.containerOK { + return nil + } + + // now try to create the container + err := f.pacer.Call(func() (bool, error) { + ctx := context.Background() + _, err := f.cntURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) + if err != nil { + if storageErr, ok := err.(azblob.StorageError); ok { + switch storageErr.ServiceCode() { + case azblob.ServiceCodeContainerAlreadyExists: + f.containerOK = true + return false, nil + case azblob.ServiceCodeContainerBeingDeleted: + f.containerDeleted = true + return true, err + } + } + } + return f.shouldRetry(err) + }) + if err == nil { + f.containerOK = true + f.containerDeleted = false + } + return errors.Wrap(err, "failed to make container") +} + +// isEmpty checks to see if a given directory is empty and returns an error if not +func (f *Fs) isEmpty(dir string) (err error) { + empty := true + err = f.list("", true, 1, func(remote string, object *azblob.BlobItem, isDirectory bool) error { + empty = false + return nil + }) + if err != nil { + return err + } + if !empty { + return fs.ErrorDirectoryNotEmpty + } + return nil +} + +// deleteContainer deletes the container. It can delete a full +// container so use isEmpty if you don't want that. +func (f *Fs) deleteContainer() error { + f.containerOKMu.Lock() + defer f.containerOKMu.Unlock() + options := azblob.ContainerAccessConditions{} + ctx := context.Background() + err := f.pacer.Call(func() (bool, error) { + _, err := f.cntURL.GetProperties(ctx, azblob.LeaseAccessConditions{}) + if err == nil { + _, err = f.cntURL.Delete(ctx, options) + } + + if err != nil { + // Check http error code along with service code, current SDK doesn't populate service code correctly sometimes + if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) { + return false, fs.ErrorDirNotFound + } + + return f.shouldRetry(err) + } + + return f.shouldRetry(err) + }) + if err == nil { + f.containerOK = false + f.containerDeleted = true + } + return errors.Wrap(err, "failed to delete container") +} + +// Rmdir deletes the container if the fs is at the root +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + err := f.isEmpty(dir) + if err != nil { + return err + } + if f.root != "" || dir != "" { + return nil + } + return f.deleteContainer() +} + +// Precision of the remote +func (f *Fs) Precision() time.Duration { + return time.Nanosecond +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +// Purge deletes all the files and directories including the old versions. +func (f *Fs) Purge() error { + dir := "" // forward compat! + if f.root != "" || dir != "" { + // Delegate to caller if not root container + return fs.ErrorCantPurge + } + return f.deleteContainer() +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + err := f.Mkdir("") + if err != nil { + return nil, err + } + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + dstBlobURL := f.getBlobReference(remote) + srcBlobURL := srcObj.getBlobReference() + + source, err := url.Parse(srcBlobURL.String()) + if err != nil { + return nil, err + } + + options := azblob.BlobAccessConditions{} + ctx := context.Background() + var startCopy *azblob.BlobStartCopyFromURLResponse + + err = f.pacer.Call(func() (bool, error) { + startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, options, options) + return f.shouldRetry(err) + }) + if err != nil { + return nil, err + } + + copyStatus := startCopy.CopyStatus() + for copyStatus == azblob.CopyStatusPending { + time.Sleep(1 * time.Second) + getMetadata, err := dstBlobURL.GetProperties(ctx, options) + if err != nil { + return nil, err + } + copyStatus = getMetadata.CopyStatus() + } + + return f.NewObject(remote) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the MD5 of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.MD5 { + return "", hash.ErrUnsupported + } + // Convert base64 encoded md5 into lower case hex + if o.md5 == "" { + return "", nil + } + data, err := base64.StdEncoding.DecodeString(o.md5) + if err != nil { + return "", errors.Wrapf(err, "Failed to decode Content-MD5: %q", o.md5) + } + return hex.EncodeToString(data), nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.size +} + +func (o *Object) setMetadata(metadata azblob.Metadata) { + if len(metadata) > 0 { + o.meta = metadata + if modTime, ok := metadata[modTimeKey]; ok { + when, err := time.Parse(timeFormatIn, modTime) + if err != nil { + fs.Debugf(o, "Couldn't parse %v = %q: %v", modTimeKey, modTime, err) + } + o.modTime = when + } + } else { + o.meta = nil + } +} + +// decodeMetaDataFromPropertiesResponse sets the metadata from the data passed in +// +// Sets +// o.id +// o.modTime +// o.size +// o.md5 +// o.meta +func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetPropertiesResponse) (err error) { + // NOTE - In BlobGetPropertiesResponse, Client library returns MD5 as base64 decoded string + // unlike BlobProperties in BlobItem (used in decodeMetadataFromBlob) which returns base64 + // encoded bytes. Object needs to maintain this as base64 encoded string. + o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5()) + o.mimeType = info.ContentType() + o.size = info.ContentLength() + o.modTime = time.Time(info.LastModified()) + o.accessTier = azblob.AccessTierType(info.AccessTier()) + o.setMetadata(info.NewMetadata()) + + return nil +} + +func (o *Object) decodeMetaDataFromBlob(info *azblob.BlobItem) (err error) { + o.md5 = string(info.Properties.ContentMD5) + o.mimeType = *info.Properties.ContentType + o.size = *info.Properties.ContentLength + o.modTime = info.Properties.LastModified + o.accessTier = info.Properties.AccessTier + o.setMetadata(info.Metadata) + return nil +} + +// getBlobReference creates an empty blob reference with no metadata +func (o *Object) getBlobReference() azblob.BlobURL { + return o.fs.getBlobReference(o.remote) +} + +// clearMetaData clears enough metadata so readMetaData will re-read it +func (o *Object) clearMetaData() { + o.modTime = time.Time{} +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// Sets +// o.id +// o.modTime +// o.size +// o.md5 +func (o *Object) readMetaData() (err error) { + if !o.modTime.IsZero() { + return nil + } + blob := o.getBlobReference() + + // Read metadata (this includes metadata) + options := azblob.BlobAccessConditions{} + ctx := context.Background() + var blobProperties *azblob.BlobGetPropertiesResponse + err = o.fs.pacer.Call(func() (bool, error) { + blobProperties, err = blob.GetProperties(ctx, options) + return o.fs.shouldRetry(err) + }) + if err != nil { + // On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well + if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeBlobNotFound || storageErr.Response().StatusCode == http.StatusNotFound) { + return fs.ErrorObjectNotFound + } + return err + } + + return o.decodeMetaDataFromPropertiesResponse(blobProperties) +} + +// timeString returns modTime as the number of milliseconds +// elapsed since January 1, 1970 UTC as a decimal string. +func timeString(modTime time.Time) string { + return strconv.FormatInt(modTime.UnixNano()/1E6, 10) +} + +// parseTimeString converts a decimal string number of milliseconds +// elapsed since January 1, 1970 UTC into a time.Time and stores it in +// the modTime variable. +func (o *Object) parseTimeString(timeString string) (err error) { + if timeString == "" { + return nil + } + unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64) + if err != nil { + fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err) + return err + } + o.modTime = time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC() + return nil +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() (result time.Time) { + // The error is logged in readMetaData + _ = o.readMetaData() + return o.modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + // Make sure o.meta is not nil + if o.meta == nil { + o.meta = make(map[string]string, 1) + } + // Set modTimeKey in it + o.meta[modTimeKey] = modTime.Format(timeFormatOut) + + blob := o.getBlobReference() + ctx := context.Background() + err := o.fs.pacer.Call(func() (bool, error) { + _, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{}) + return o.fs.shouldRetry(err) + }) + if err != nil { + return err + } + o.modTime = modTime + return nil +} + +// Storable returns if this object is storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + // Offset and Count for range download + var offset int64 + var count int64 + if o.AccessTier() == azblob.AccessTierArchive { + return nil, errors.Errorf("Blob in archive tier, you need to set tier to hot or cool first") + } + + for _, option := range options { + switch x := option.(type) { + case *fs.RangeOption: + offset, count = x.Decode(o.size) + if count < 0 { + count = o.size - offset + } + case *fs.SeekOption: + offset = x.Offset + default: + if option.Mandatory() { + fs.Logf(o, "Unsupported mandatory option: %v", option) + } + } + } + blob := o.getBlobReference() + ctx := context.Background() + ac := azblob.BlobAccessConditions{} + var dowloadResponse *azblob.DownloadResponse + err = o.fs.pacer.Call(func() (bool, error) { + dowloadResponse, err = blob.Download(ctx, offset, count, ac, false) + return o.fs.shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to open for download") + } + in = dowloadResponse.Body(azblob.RetryReaderOptions{}) + return in, nil +} + +// dontEncode is the characters that do not need percent-encoding +// +// The characters that do not need percent-encoding are a subset of +// the printable ASCII characters: upper-case letters, lower-case +// letters, digits, ".", "_", "-", "/", "~", "!", "$", "'", "(", ")", +// "*", ";", "=", ":", and "@". All other byte values in a UTF-8 must +// be replaced with "%" and the two-digit hex value of the byte. +const dontEncode = (`abcdefghijklmnopqrstuvwxyz` + + `ABCDEFGHIJKLMNOPQRSTUVWXYZ` + + `0123456789` + + `._-/~!$'()*;=:@`) + +// noNeedToEncode is a bitmap of characters which don't need % encoding +var noNeedToEncode [256]bool + +func init() { + for _, c := range dontEncode { + noNeedToEncode[c] = true + } +} + +// readSeeker joins an io.Reader and an io.Seeker +type readSeeker struct { + io.Reader + io.Seeker +} + +// uploadMultipart uploads a file using multipart upload +// +// Write a larger blob, using CreateBlockBlob, PutBlock, and PutBlockList. +func (o *Object) uploadMultipart(in io.Reader, size int64, blob *azblob.BlobURL, httpHeaders *azblob.BlobHTTPHeaders) (err error) { + // Calculate correct chunkSize + chunkSize := int64(o.fs.opt.ChunkSize) + var totalParts int64 + for { + // Calculate number of parts + var remainder int64 + totalParts, remainder = size/chunkSize, size%chunkSize + if remainder != 0 { + totalParts++ + } + if totalParts < maxTotalParts { + break + } + // Double chunk size if the number of parts is too big + chunkSize *= 2 + if chunkSize > int64(maxChunkSize) { + return errors.Errorf("can't upload as it is too big %v - takes more than %d chunks of %v", fs.SizeSuffix(size), totalParts, fs.SizeSuffix(chunkSize/2)) + } + } + fs.Debugf(o, "Multipart upload session started for %d parts of size %v", totalParts, fs.SizeSuffix(chunkSize)) + + // https://godoc.org/github.com/Azure/azure-storage-blob-go/2017-07-29/azblob#example-BlockBlobURL + // Utilities are cloned from above example + // These helper functions convert a binary block ID to a base-64 string and vice versa + // NOTE: The blockID must be <= 64 bytes and ALL blockIDs for the block must be the same length + blockIDBinaryToBase64 := func(blockID []byte) string { return base64.StdEncoding.EncodeToString(blockID) } + // These helper functions convert an int block ID to a base-64 string and vice versa + blockIDIntToBase64 := func(blockID uint64) string { + binaryBlockID := (&[8]byte{})[:] // All block IDs are 8 bytes long + binary.LittleEndian.PutUint64(binaryBlockID, blockID) + return blockIDBinaryToBase64(binaryBlockID) + } + + // block ID variables + var ( + rawID uint64 + blockID = "" // id in base64 encoded form + blocks []string + ) + + // increment the blockID + nextID := func() { + rawID++ + blockID = blockIDIntToBase64(rawID) + blocks = append(blocks, blockID) + } + + // Get BlockBlobURL, we will use default pipeline here + blockBlobURL := blob.ToBlockBlobURL() + ctx := context.Background() + ac := azblob.LeaseAccessConditions{} // Use default lease access conditions + + // unwrap the accounting from the input, we use wrap to put it + // back on after the buffering + in, wrap := accounting.UnWrap(in) + + // Upload the chunks + remaining := size + position := int64(0) + errs := make(chan error, 1) + var wg sync.WaitGroup +outer: + for part := 0; part < int(totalParts); part++ { + // Check any errors + select { + case err = <-errs: + break outer + default: + } + + reqSize := remaining + if reqSize >= chunkSize { + reqSize = chunkSize + } + + // Make a block of memory + buf := make([]byte, reqSize) + + // Read the chunk + _, err = io.ReadFull(in, buf) + if err != nil { + err = errors.Wrap(err, "multipart upload failed to read source") + break outer + } + + // Transfer the chunk + nextID() + wg.Add(1) + o.fs.uploadToken.Get() + go func(part int, position int64, blockID string) { + defer wg.Done() + defer o.fs.uploadToken.Put() + fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, totalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize)) + + err = o.fs.pacer.Call(func() (bool, error) { + bufferReader := bytes.NewReader(buf) + wrappedReader := wrap(bufferReader) + rs := readSeeker{wrappedReader, bufferReader} + _, err = blockBlobURL.StageBlock(ctx, blockID, &rs, ac) + return o.fs.shouldRetry(err) + }) + + if err != nil { + err = errors.Wrap(err, "multipart upload failed to upload part") + select { + case errs <- err: + default: + } + return + } + }(part, position, blockID) + + // ready for next block + remaining -= chunkSize + position += chunkSize + } + wg.Wait() + if err == nil { + select { + case err = <-errs: + default: + } + } + if err != nil { + return err + } + + // Finalise the upload session + err = o.fs.pacer.Call(func() (bool, error) { + _, err := blockBlobURL.CommitBlockList(ctx, blocks, *httpHeaders, o.meta, azblob.BlobAccessConditions{}) + return o.fs.shouldRetry(err) + }) + if err != nil { + return errors.Wrap(err, "multipart upload failed to finalize") + } + return nil +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + err = o.fs.Mkdir("") + if err != nil { + return err + } + size := src.Size() + // Update Mod time + o.updateMetadataWithModTime(src.ModTime()) + if err != nil { + return err + } + + blob := o.getBlobReference() + httpHeaders := azblob.BlobHTTPHeaders{} + httpHeaders.ContentType = fs.MimeType(o) + // Multipart upload doesn't support MD5 checksums at put block calls, hence calculate + // MD5 only for PutBlob requests + if size < int64(o.fs.opt.UploadCutoff) { + if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" { + sourceMD5bytes, err := hex.DecodeString(sourceMD5) + if err == nil { + httpHeaders.ContentMD5 = sourceMD5bytes + } else { + fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err) + } + } + } + + putBlobOptions := azblob.UploadStreamToBlockBlobOptions{ + BufferSize: int(o.fs.opt.ChunkSize), + MaxBuffers: 4, + Metadata: o.meta, + BlobHTTPHeaders: httpHeaders, + } + + ctx := context.Background() + // Don't retry, return a retry error instead + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + if size >= int64(o.fs.opt.UploadCutoff) { + // If a large file upload in chunks + err = o.uploadMultipart(in, size, &blob, &httpHeaders) + } else { + // Write a small blob in one transaction + blockBlobURL := blob.ToBlockBlobURL() + _, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions) + } + return o.fs.shouldRetry(err) + }) + if err != nil { + return err + } + // Refresh metadata on object + o.clearMetaData() + err = o.readMetaData() + if err != nil { + return err + } + + // If tier is not changed or not specified, do not attempt to invoke `SetBlobTier` operation + if o.fs.opt.AccessTier == string(defaultAccessTier) || o.fs.opt.AccessTier == string(o.AccessTier()) { + return nil + } + + // Now, set blob tier based on configured access tier + desiredAccessTier := azblob.AccessTierType(o.fs.opt.AccessTier) + err = o.fs.pacer.Call(func() (bool, error) { + _, err := blob.SetTier(ctx, desiredAccessTier) + return o.fs.shouldRetry(err) + }) + + if err != nil { + return errors.Wrap(err, "Failed to set Blob Tier") + } + return nil +} + +// Remove an object +func (o *Object) Remove() error { + blob := o.getBlobReference() + snapShotOptions := azblob.DeleteSnapshotsOptionNone + ac := azblob.BlobAccessConditions{} + ctx := context.Background() + return o.fs.pacer.Call(func() (bool, error) { + _, err := blob.Delete(ctx, snapShotOptions, ac) + return o.fs.shouldRetry(err) + }) +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + return o.mimeType +} + +// AccessTier of an object, default is of type none +func (o *Object) AccessTier() azblob.AccessTierType { + return o.accessTier +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.Copier = &Fs{} + _ fs.Purger = &Fs{} + _ fs.ListRer = &Fs{} + _ fs.Object = &Object{} + _ fs.MimeTyper = &Object{} +) diff --git a/.rclone_repo/backend/azureblob/azureblob_test.go b/.rclone_repo/backend/azureblob/azureblob_test.go new file mode 100755 index 0000000..3a36d71 --- /dev/null +++ b/.rclone_repo/backend/azureblob/azureblob_test.go @@ -0,0 +1,20 @@ +// Test AzureBlob filesystem interface + +// +build !freebsd,!netbsd,!openbsd,!plan9,!solaris,go1.8 + +package azureblob_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/azureblob" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestAzureBlob:", + NilObject: (*azureblob.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/azureblob/azureblob_unsupported.go b/.rclone_repo/backend/azureblob/azureblob_unsupported.go new file mode 100755 index 0000000..5494798 --- /dev/null +++ b/.rclone_repo/backend/azureblob/azureblob_unsupported.go @@ -0,0 +1,6 @@ +// Build for azureblob for unsupported platforms to stop go complaining +// about "no buildable Go source files " + +// +build freebsd netbsd openbsd plan9 solaris !go1.8 + +package azureblob diff --git a/.rclone_repo/backend/b2/api/types.go b/.rclone_repo/backend/b2/api/types.go new file mode 100755 index 0000000..501647e --- /dev/null +++ b/.rclone_repo/backend/b2/api/types.go @@ -0,0 +1,312 @@ +package api + +import ( + "fmt" + "path" + "strconv" + "strings" + "time" + + "github.com/ncw/rclone/fs/fserrors" +) + +// Error describes a B2 error response +type Error struct { + Status int `json:"status"` // The numeric HTTP status code. Always matches the status in the HTTP response. + Code string `json:"code"` // A single-identifier code that identifies the error. + Message string `json:"message"` // A human-readable message, in English, saying what went wrong. +} + +// Error statisfies the error interface +func (e *Error) Error() string { + return fmt.Sprintf("%s (%d %s)", e.Message, e.Status, e.Code) +} + +// Fatal statisfies the Fatal interface +// +// It indicates which errors should be treated as fatal +func (e *Error) Fatal() bool { + return e.Status == 403 // 403 errors shouldn't be retried +} + +var _ fserrors.Fataler = (*Error)(nil) + +// Bucket describes a B2 bucket +type Bucket struct { + ID string `json:"bucketId"` + AccountID string `json:"accountId"` + Name string `json:"bucketName"` + Type string `json:"bucketType"` +} + +// Timestamp is a UTC time when this file was uploaded. It is a base +// 10 number of milliseconds since midnight, January 1, 1970 UTC. This +// fits in a 64 bit integer such as the type "long" in the programming +// language Java. It is intended to be compatible with Java's time +// long. For example, it can be passed directly into the java call +// Date.setTime(long time). +type Timestamp time.Time + +// MarshalJSON turns a Timestamp into JSON (in UTC) +func (t *Timestamp) MarshalJSON() (out []byte, err error) { + timestamp := (*time.Time)(t).UTC().UnixNano() + return []byte(strconv.FormatInt(timestamp/1E6, 10)), nil +} + +// UnmarshalJSON turns JSON into a Timestamp +func (t *Timestamp) UnmarshalJSON(data []byte) error { + timestamp, err := strconv.ParseInt(string(data), 10, 64) + if err != nil { + return err + } + *t = Timestamp(time.Unix(timestamp/1E3, (timestamp%1E3)*1E6).UTC()) + return nil +} + +const versionFormat = "-v2006-01-02-150405.000" + +// AddVersion adds the timestamp as a version string into the filename passed in. +func (t Timestamp) AddVersion(remote string) string { + ext := path.Ext(remote) + base := remote[:len(remote)-len(ext)] + s := time.Time(t).Format(versionFormat) + // Replace the '.' with a '-' + s = strings.Replace(s, ".", "-", -1) + return base + s + ext +} + +// RemoveVersion removes the timestamp from a filename as a version string. +// +// It returns the new file name and a timestamp, or the old filename +// and a zero timestamp. +func RemoveVersion(remote string) (t Timestamp, newRemote string) { + newRemote = remote + ext := path.Ext(remote) + base := remote[:len(remote)-len(ext)] + if len(base) < len(versionFormat) { + return + } + versionStart := len(base) - len(versionFormat) + // Check it ends in -xxx + if base[len(base)-4] != '-' { + return + } + // Replace with .xxx for parsing + base = base[:len(base)-4] + "." + base[len(base)-3:] + newT, err := time.Parse(versionFormat, base[versionStart:]) + if err != nil { + return + } + return Timestamp(newT), base[:versionStart] + ext +} + +// IsZero returns true if the timestamp is unitialised +func (t Timestamp) IsZero() bool { + return time.Time(t).IsZero() +} + +// Equal compares two timestamps +// +// If either are !IsZero then it returns false +func (t Timestamp) Equal(s Timestamp) bool { + if time.Time(t).IsZero() { + return false + } + if time.Time(s).IsZero() { + return false + } + return time.Time(t).Equal(time.Time(s)) +} + +// File is info about a file +type File struct { + ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version. + Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name. + Action string `json:"action"` // Either "upload" or "hide". "upload" means a file that was uploaded to B2 Cloud Storage. "hide" means a file version marking the file as hidden, so that it will not show up in b2_list_file_names. The result of b2_list_file_names will contain only "upload". The result of b2_list_file_versions may have both. + Size int64 `json:"size"` // The number of bytes in the file. + UploadTimestamp Timestamp `json:"uploadTimestamp"` // This is a UTC time when this file was uploaded. + SHA1 string `json:"contentSha1"` // The SHA1 of the bytes stored in the file. + ContentType string `json:"contentType"` // The MIME type of the file. + Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file. +} + +// AuthorizeAccountResponse is as returned from the b2_authorize_account call +type AuthorizeAccountResponse struct { + AbsoluteMinimumPartSize int `json:"absoluteMinimumPartSize"` // The smallest possible size of a part of a large file. + AccountID string `json:"accountId"` // The identifier for the account. + Allowed struct { // An object (see below) containing the capabilities of this auth token, and any restrictions on using it. + BucketID string `json:"bucketId"` // When present, access is restricted to one bucket. + Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has. + NamePrefix interface{} `json:"namePrefix"` // When present, access is restricted to files whose names start with the prefix + } `json:"allowed"` + APIURL string `json:"apiUrl"` // The base URL to use for all API calls except for uploading and downloading files. + AuthorizationToken string `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header. + DownloadURL string `json:"downloadUrl"` // The base URL to use for downloading files. + MinimumPartSize int `json:"minimumPartSize"` // DEPRECATED: This field will always have the same value as recommendedPartSize. Use recommendedPartSize instead. + RecommendedPartSize int `json:"recommendedPartSize"` // The recommended size for each part of a large file. We recommend using this part size for optimal upload performance. +} + +// ListBucketsRequest is parameters for b2_list_buckets call +type ListBucketsRequest struct { + AccountID string `json:"accountId"` // The identifier for the account. + BucketID string `json:"bucketId,omitempty"` // When specified, the result will be a list containing just this bucket. + BucketName string `json:"bucketName,omitempty"` // When specified, the result will be a list containing just this bucket. + BucketTypes []string `json:"bucketTypes,omitempty"` // If present, B2 will use it as a filter for bucket types returned in the list buckets response. +} + +// ListBucketsResponse is as returned from the b2_list_buckets call +type ListBucketsResponse struct { + Buckets []Bucket `json:"buckets"` +} + +// ListFileNamesRequest is as passed to b2_list_file_names or b2_list_file_versions +type ListFileNamesRequest struct { + BucketID string `json:"bucketId"` // required - The bucket to look for file names in. + StartFileName string `json:"startFileName,omitempty"` // optional - The first file name to return. If there is a file with this name, it will be returned in the list. If not, the first file name after this the first one after this name. + MaxFileCount int `json:"maxFileCount,omitempty"` // optional - The maximum number of files to return from this call. The default value is 100, and the maximum allowed is 1000. + StartFileID string `json:"startFileId,omitempty"` // optional - What to pass in to startFileId for the next search to continue where this one left off. + Prefix string `json:"prefix,omitempty"` // optional - Files returned will be limited to those with the given prefix. Defaults to the empty string, which matches all files. + Delimiter string `json:"delimiter,omitempty"` // Files returned will be limited to those within the top folder, or any one subfolder. Defaults to NULL. Folder names will also be returned. The delimiter character will be used to "break" file names into folders. +} + +// ListFileNamesResponse is as received from b2_list_file_names or b2_list_file_versions +type ListFileNamesResponse struct { + Files []File `json:"files"` // An array of objects, each one describing one file. + NextFileName *string `json:"nextFileName"` // What to pass in to startFileName for the next search to continue where this one left off, or null if there are no more files. + NextFileID *string `json:"nextFileId"` // What to pass in to startFileId for the next search to continue where this one left off, or null if there are no more files. +} + +// GetUploadURLRequest is passed to b2_get_upload_url +type GetUploadURLRequest struct { + BucketID string `json:"bucketId"` // The ID of the bucket that you want to upload to. +} + +// GetUploadURLResponse is received from b2_get_upload_url +type GetUploadURLResponse struct { + BucketID string `json:"bucketId"` // The unique ID of the bucket. + UploadURL string `json:"uploadUrl"` // The URL that can be used to upload files to this bucket, see b2_upload_file. + AuthorizationToken string `json:"authorizationToken"` // The authorizationToken that must be used when uploading files to this bucket, see b2_upload_file. +} + +// FileInfo is received from b2_upload_file, b2_get_file_info and b2_finish_large_file +type FileInfo struct { + ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version. + Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name. + Action string `json:"action"` // Either "upload" or "hide". "upload" means a file that was uploaded to B2 Cloud Storage. "hide" means a file version marking the file as hidden, so that it will not show up in b2_list_file_names. The result of b2_list_file_names will contain only "upload". The result of b2_list_file_versions may have both. + AccountID string `json:"accountId"` // Your account ID. + BucketID string `json:"bucketId"` // The bucket that the file is in. + Size int64 `json:"contentLength"` // The number of bytes stored in the file. + UploadTimestamp Timestamp `json:"uploadTimestamp"` // This is a UTC time when this file was uploaded. + SHA1 string `json:"contentSha1"` // The SHA1 of the bytes stored in the file. + ContentType string `json:"contentType"` // The MIME type of the file. + Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file. +} + +// CreateBucketRequest is used to create a bucket +type CreateBucketRequest struct { + AccountID string `json:"accountId"` + Name string `json:"bucketName"` + Type string `json:"bucketType"` +} + +// DeleteBucketRequest is used to create a bucket +type DeleteBucketRequest struct { + ID string `json:"bucketId"` + AccountID string `json:"accountId"` +} + +// DeleteFileRequest is used to delete a file version +type DeleteFileRequest struct { + ID string `json:"fileId"` // The ID of the file, as returned by b2_upload_file, b2_list_file_names, or b2_list_file_versions. + Name string `json:"fileName"` // The name of this file. +} + +// HideFileRequest is used to delete a file +type HideFileRequest struct { + BucketID string `json:"bucketId"` // The bucket containing the file to hide. + Name string `json:"fileName"` // The name of the file to hide. +} + +// GetFileInfoRequest is used to return a FileInfo struct with b2_get_file_info +type GetFileInfoRequest struct { + ID string `json:"fileId"` // The ID of the file, as returned by b2_upload_file, b2_list_file_names, or b2_list_file_versions. +} + +// StartLargeFileRequest (b2_start_large_file) Prepares for uploading the parts of a large file. +// +// If the original source of the file being uploaded has a last +// modified time concept, Backblaze recommends using +// src_last_modified_millis as the name, and a string holding the base +// 10 number number of milliseconds since midnight, January 1, 1970 +// UTC. This fits in a 64 bit integer such as the type "long" in the +// programming language Java. It is intended to be compatible with +// Java's time long. For example, it can be passed directly into the +// Java call Date.setTime(long time). +// +// If the caller knows the SHA1 of the entire large file being +// uploaded, Backblaze recommends using large_file_sha1 as the name, +// and a 40 byte hex string representing the SHA1. +// +// Example: { "src_last_modified_millis" : "1452802803026", "large_file_sha1" : "a3195dc1e7b46a2ff5da4b3c179175b75671e80d", "color": "blue" } +type StartLargeFileRequest struct { + BucketID string `json:"bucketId"` //The ID of the bucket that the file will go in. + Name string `json:"fileName"` // The name of the file. See Files for requirements on file names. + ContentType string `json:"contentType"` // The MIME type of the content of the file, which will be returned in the Content-Type header when downloading the file. Use the Content-Type b2/x-auto to automatically set the stored Content-Type post upload. In the case where a file extension is absent or the lookup fails, the Content-Type is set to application/octet-stream. + Info map[string]string `json:"fileInfo"` // A JSON object holding the name/value pairs for the custom file info. +} + +// StartLargeFileResponse is the response to StartLargeFileRequest +type StartLargeFileResponse struct { + ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version. + Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name. + AccountID string `json:"accountId"` // The identifier for the account. + BucketID string `json:"bucketId"` // The unique ID of the bucket. + ContentType string `json:"contentType"` // The MIME type of the file. + Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file. + UploadTimestamp Timestamp `json:"uploadTimestamp"` // This is a UTC time when this file was uploaded. +} + +// GetUploadPartURLRequest is passed to b2_get_upload_part_url +type GetUploadPartURLRequest struct { + ID string `json:"fileId"` // The unique identifier of the file being uploaded. +} + +// GetUploadPartURLResponse is received from b2_get_upload_url +type GetUploadPartURLResponse struct { + ID string `json:"fileId"` // The unique identifier of the file being uploaded. + UploadURL string `json:"uploadUrl"` // The URL that can be used to upload files to this bucket, see b2_upload_part. + AuthorizationToken string `json:"authorizationToken"` // The authorizationToken that must be used when uploading files to this bucket, see b2_upload_part. +} + +// UploadPartResponse is the response to b2_upload_part +type UploadPartResponse struct { + ID string `json:"fileId"` // The unique identifier of the file being uploaded. + PartNumber int64 `json:"partNumber"` // Which part this is (starting from 1) + Size int64 `json:"contentLength"` // The number of bytes stored in the file. + SHA1 string `json:"contentSha1"` // The SHA1 of the bytes stored in the file. +} + +// FinishLargeFileRequest is passed to b2_finish_large_file +// +// The response is a FileInfo object (with extra AccountID and BucketID fields which we ignore). +// +// Large files do not have a SHA1 checksum. The value will always be "none". +type FinishLargeFileRequest struct { + ID string `json:"fileId"` // The unique identifier of the file being uploaded. + SHA1s []string `json:"partSha1Array"` // A JSON array of hex SHA1 checksums of the parts of the large file. This is a double-check that the right parts were uploaded in the right order, and that none were missed. Note that the part numbers start at 1, and the SHA1 of the part 1 is the first string in the array, at index 0. +} + +// CancelLargeFileRequest is passed to b2_finish_large_file +// +// The response is a CancelLargeFileResponse +type CancelLargeFileRequest struct { + ID string `json:"fileId"` // The unique identifier of the file being uploaded. +} + +// CancelLargeFileResponse is the response to CancelLargeFileRequest +type CancelLargeFileResponse struct { + ID string `json:"fileId"` // The unique identifier of the file being uploaded. + Name string `json:"fileName"` // The name of this file. + AccountID string `json:"accountId"` // The identifier for the account. + BucketID string `json:"bucketId"` // The unique ID of the bucket. +} diff --git a/.rclone_repo/backend/b2/api/types_test.go b/.rclone_repo/backend/b2/api/types_test.go new file mode 100755 index 0000000..6131a2b --- /dev/null +++ b/.rclone_repo/backend/b2/api/types_test.go @@ -0,0 +1,87 @@ +package api_test + +import ( + "testing" + "time" + + "github.com/ncw/rclone/backend/b2/api" + "github.com/ncw/rclone/fstest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + emptyT api.Timestamp + t0 = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123456789Z")) + t0r = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123000000Z")) + t1 = api.Timestamp(fstest.Time("2001-02-03T04:05:06.123000000Z")) +) + +func TestTimestampMarshalJSON(t *testing.T) { + resB, err := t0.MarshalJSON() + res := string(resB) + require.NoError(t, err) + assert.Equal(t, "3661123", res) + + resB, err = t1.MarshalJSON() + res = string(resB) + require.NoError(t, err) + assert.Equal(t, "981173106123", res) +} + +func TestTimestampUnmarshalJSON(t *testing.T) { + var tActual api.Timestamp + err := tActual.UnmarshalJSON([]byte("981173106123")) + require.NoError(t, err) + assert.Equal(t, (time.Time)(t1), (time.Time)(tActual)) +} + +func TestTimestampAddVersion(t *testing.T) { + for _, test := range []struct { + t api.Timestamp + in string + expected string + }{ + {t0, "potato.txt", "potato-v1970-01-01-010101-123.txt"}, + {t1, "potato", "potato-v2001-02-03-040506-123"}, + {t1, "", "-v2001-02-03-040506-123"}, + } { + actual := test.t.AddVersion(test.in) + assert.Equal(t, test.expected, actual, test.in) + } +} + +func TestTimestampRemoveVersion(t *testing.T) { + for _, test := range []struct { + in string + expectedT api.Timestamp + expectedRemote string + }{ + {"potato.txt", emptyT, "potato.txt"}, + {"potato-v1970-01-01-010101-123.txt", t0r, "potato.txt"}, + {"potato-v2001-02-03-040506-123", t1, "potato"}, + {"-v2001-02-03-040506-123", t1, ""}, + {"potato-v2A01-02-03-040506-123", emptyT, "potato-v2A01-02-03-040506-123"}, + {"potato-v2001-02-03-040506=123", emptyT, "potato-v2001-02-03-040506=123"}, + } { + actualT, actualRemote := api.RemoveVersion(test.in) + assert.Equal(t, test.expectedT, actualT, test.in) + assert.Equal(t, test.expectedRemote, actualRemote, test.in) + } +} + +func TestTimestampIsZero(t *testing.T) { + assert.True(t, emptyT.IsZero()) + assert.False(t, t0.IsZero()) + assert.False(t, t1.IsZero()) +} + +func TestTimestampEqual(t *testing.T) { + assert.False(t, emptyT.Equal(emptyT)) + assert.False(t, t0.Equal(emptyT)) + assert.False(t, emptyT.Equal(t0)) + assert.False(t, t0.Equal(t1)) + assert.False(t, t1.Equal(t0)) + assert.True(t, t0.Equal(t0)) + assert.True(t, t1.Equal(t1)) +} diff --git a/.rclone_repo/backend/b2/b2.go b/.rclone_repo/backend/b2/b2.go new file mode 100755 index 0000000..ab9c6db --- /dev/null +++ b/.rclone_repo/backend/b2/b2.go @@ -0,0 +1,1485 @@ +// Package b2 provides an interface to the Backblaze B2 object storage system +package b2 + +// FIXME should we remove sha1 checks from here as rclone now supports +// checking SHA1s? + +import ( + "bufio" + "bytes" + "crypto/sha1" + "fmt" + gohash "hash" + "io" + "net/http" + "path" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/ncw/rclone/backend/b2/api" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/accounting" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/fs/walk" + "github.com/ncw/rclone/lib/pacer" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" +) + +const ( + defaultEndpoint = "https://api.backblazeb2.com" + headerPrefix = "x-bz-info-" // lower case as that is what the server returns + timeKey = "src_last_modified_millis" + timeHeader = headerPrefix + timeKey + sha1Key = "large_file_sha1" + sha1Header = "X-Bz-Content-Sha1" + sha1InfoHeader = headerPrefix + sha1Key + testModeHeader = "X-Bz-Test-Mode" + retryAfterHeader = "Retry-After" + minSleep = 10 * time.Millisecond + maxSleep = 5 * time.Minute + decayConstant = 1 // bigger for slower decay, exponential + maxParts = 10000 + maxVersions = 100 // maximum number of versions we search in --b2-versions mode + minChunkSize = 5E6 + defaultChunkSize = 96 * 1024 * 1024 + defaultUploadCutoff = 200E6 +) + +// Globals +var ( + errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode") +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "b2", + Description: "Backblaze B2", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "account", + Help: "Account ID or Application Key ID", + Required: true, + }, { + Name: "key", + Help: "Application Key", + Required: true, + }, { + Name: "endpoint", + Help: "Endpoint for the service.\nLeave blank normally.", + Advanced: true, + }, { + Name: "test_mode", + Help: "A flag string for X-Bz-Test-Mode header for debugging.", + Default: "", + Hide: fs.OptionHideConfigurator, + Advanced: true, + }, { + Name: "versions", + Help: "Include old versions in directory listings.", + Default: false, + Advanced: true, + }, { + Name: "hard_delete", + Help: "Permanently delete files on remote removal, otherwise hide files.", + Default: false, + }, { + Name: "upload_cutoff", + Help: "Cutoff for switching to chunked upload.", + Default: fs.SizeSuffix(defaultUploadCutoff), + Advanced: true, + }, { + Name: "chunk_size", + Help: "Upload chunk size. Must fit in memory.", + Default: fs.SizeSuffix(defaultChunkSize), + Advanced: true, + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + Account string `config:"account"` + Key string `config:"key"` + Endpoint string `config:"endpoint"` + TestMode string `config:"test_mode"` + Versions bool `config:"versions"` + HardDelete bool `config:"hard_delete"` + UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` +} + +// Fs represents a remote b2 server +type Fs struct { + name string // name of this remote + root string // the path we are working on if any + opt Options // parsed config options + features *fs.Features // optional features + srv *rest.Client // the connection to the b2 server + bucket string // the bucket we are working on + bucketOKMu sync.Mutex // mutex to protect bucket OK + bucketOK bool // true if we have created the bucket + bucketIDMutex sync.Mutex // mutex to protect _bucketID + _bucketID string // the ID of the bucket we are working on + info api.AuthorizeAccountResponse // result of authorize call + uploadMu sync.Mutex // lock for upload variable + uploads []*api.GetUploadURLResponse // result of get upload URL calls + authMu sync.Mutex // lock for authorizing the account + pacer *pacer.Pacer // To pace and retry the API calls + bufferTokens chan []byte // control concurrency of multipart uploads +} + +// Object describes a b2 object +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + id string // b2 id of the file + modTime time.Time // The modified time of the object if known + sha1 string // SHA-1 hash if known + size int64 // Size of the object + mimeType string // Content-Type of the object +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + if f.root == "" { + return f.bucket + } + return f.bucket + "/" + f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + if f.root == "" { + return fmt.Sprintf("B2 bucket %s", f.bucket) + } + return fmt.Sprintf("B2 bucket %s path %s", f.bucket, f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Pattern to match a b2 path +var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`) + +// parseParse parses a b2 'url' +func parsePath(path string) (bucket, directory string, err error) { + parts := matcher.FindStringSubmatch(path) + if parts == nil { + err = errors.Errorf("couldn't find bucket in b2 path %q", path) + } else { + bucket, directory = parts[1], parts[2] + directory = strings.Trim(directory, "/") + } + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 401, // Unauthorized (eg "Token has expired") + 408, // Request Timeout + 429, // Rate exceeded. + 500, // Get occasional 500 Internal Server Error + 503, // Service Unavailable + 504, // Gateway Time-out +} + +// shouldRetryNoAuth returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) { + // For 429 or 503 errors look at the Retry-After: header and + // set the retry appropriately, starting with a minimum of 1 + // second if it isn't set. + if resp != nil && (resp.StatusCode == 429 || resp.StatusCode == 503) { + var retryAfter = 1 + retryAfterString := resp.Header.Get(retryAfterHeader) + if retryAfterString != "" { + var err error + retryAfter, err = strconv.Atoi(retryAfterString) + if err != nil { + fs.Errorf(f, "Malformed %s header %q: %v", retryAfterHeader, retryAfterString, err) + } + } + retryAfterDuration := time.Duration(retryAfter) * time.Second + if f.pacer.GetSleep() < retryAfterDuration { + fs.Debugf(f, "Setting sleep to %v after error: %v", retryAfterDuration, err) + // We set 1/2 the value here because the pacer will double it immediately + f.pacer.SetSleep(retryAfterDuration / 2) + } + return true, err + } + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { + if resp != nil && resp.StatusCode == 401 { + fs.Debugf(f, "Unauthorized: %v", err) + // Reauth + authErr := f.authorizeAccount() + if authErr != nil { + err = authErr + } + return true, err + } + return f.shouldRetryNoReauth(resp, err) +} + +// errorHandler parses a non 2xx error response into an error +func errorHandler(resp *http.Response) error { + // Decode error response + errResponse := new(api.Error) + err := rest.DecodeJSON(resp, &errResponse) + if err != nil { + fs.Debugf(nil, "Couldn't decode error response: %v", err) + } + if errResponse.Code == "" { + errResponse.Code = "unknown" + } + if errResponse.Status == 0 { + errResponse.Status = resp.StatusCode + } + if errResponse.Message == "" { + errResponse.Message = "Unknown " + resp.Status + } + return errResponse +} + +// NewFs contstructs an Fs from the path, bucket:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + if opt.UploadCutoff < opt.ChunkSize { + return nil, errors.Errorf("b2: upload cutoff (%v) must be greater than or equal to chunk size (%v)", opt.UploadCutoff, opt.ChunkSize) + } + if opt.ChunkSize < minChunkSize { + return nil, errors.Errorf("b2: chunk size can't be less than %v - was %v", minChunkSize, opt.ChunkSize) + } + bucket, directory, err := parsePath(root) + if err != nil { + return nil, err + } + if opt.Account == "" { + return nil, errors.New("account not found") + } + if opt.Key == "" { + return nil, errors.New("key not found") + } + if opt.Endpoint == "" { + opt.Endpoint = defaultEndpoint + } + f := &Fs{ + name: name, + opt: *opt, + bucket: bucket, + root: directory, + srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler), + pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), + bufferTokens: make(chan []byte, fs.Config.Transfers), + } + f.features = (&fs.Features{ + ReadMimeType: true, + WriteMimeType: true, + BucketBased: true, + }).Fill(f) + // Set the test flag if required + if opt.TestMode != "" { + testMode := strings.TrimSpace(opt.TestMode) + f.srv.SetHeader(testModeHeader, testMode) + fs.Debugf(f, "Setting test header \"%s: %s\"", testModeHeader, testMode) + } + // Fill up the buffer tokens + for i := 0; i < fs.Config.Transfers; i++ { + f.bufferTokens <- nil + } + err = f.authorizeAccount() + if err != nil { + return nil, errors.Wrap(err, "failed to authorize account") + } + // If this is a key limited to a single bucket, it must exist already + if f.bucket != "" && f.info.Allowed.BucketID != "" { + f.markBucketOK() + f.setBucketID(f.info.Allowed.BucketID) + } + if f.root != "" { + f.root += "/" + // Check to see if the (bucket,directory) is actually an existing file + oldRoot := f.root + remote := path.Base(directory) + f.root = path.Dir(directory) + if f.root == "." { + f.root = "" + } else { + f.root += "/" + } + _, err := f.NewObject(remote) + if err != nil { + if err == fs.ErrorObjectNotFound { + // File doesn't exist so return old f + f.root = oldRoot + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + return f, nil +} + +// authorizeAccount gets the API endpoint and auth token. Can be used +// for reauthentication too. +func (f *Fs) authorizeAccount() error { + f.authMu.Lock() + defer f.authMu.Unlock() + opts := rest.Opts{ + Method: "GET", + Path: "/b2api/v1/b2_authorize_account", + RootURL: f.opt.Endpoint, + UserName: f.opt.Account, + Password: f.opt.Key, + ExtraHeaders: map[string]string{"Authorization": ""}, // unset the Authorization for this request + } + err := f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(&opts, nil, &f.info) + return f.shouldRetryNoReauth(resp, err) + }) + if err != nil { + return errors.Wrap(err, "failed to authenticate") + } + f.srv.SetRoot(f.info.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken) + return nil +} + +// getUploadURL returns the upload info with the UploadURL and the AuthorizationToken +// +// This should be returned with returnUploadURL when finished +func (f *Fs) getUploadURL() (upload *api.GetUploadURLResponse, err error) { + f.uploadMu.Lock() + defer f.uploadMu.Unlock() + bucketID, err := f.getBucketID() + if err != nil { + return nil, err + } + if len(f.uploads) == 0 { + opts := rest.Opts{ + Method: "POST", + Path: "/b2_get_upload_url", + } + var request = api.GetUploadURLRequest{ + BucketID: bucketID, + } + err := f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(&opts, &request, &upload) + return f.shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to get upload URL") + } + } else { + upload, f.uploads = f.uploads[0], f.uploads[1:] + } + return upload, nil +} + +// returnUploadURL returns the UploadURL to the cache +func (f *Fs) returnUploadURL(upload *api.GetUploadURLResponse) { + if upload == nil { + return + } + f.uploadMu.Lock() + f.uploads = append(f.uploads, upload) + f.uploadMu.Unlock() +} + +// clearUploadURL clears the current UploadURL and the AuthorizationToken +func (f *Fs) clearUploadURL() { + f.uploadMu.Lock() + f.uploads = nil + f.uploadMu.Unlock() +} + +// getUploadBlock gets a block from the pool of size chunkSize +func (f *Fs) getUploadBlock() []byte { + buf := <-f.bufferTokens + if buf == nil { + buf = make([]byte, f.opt.ChunkSize) + } + // fs.Debugf(f, "Getting upload block %p", buf) + return buf +} + +// putUploadBlock returns a block to the pool of size chunkSize +func (f *Fs) putUploadBlock(buf []byte) { + buf = buf[:cap(buf)] + if len(buf) != int(f.opt.ChunkSize) { + panic("bad blocksize returned to pool") + } + // fs.Debugf(f, "Returning upload block %p", buf) + f.bufferTokens <- buf +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *api.File) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + if info != nil { + err := o.decodeMetaData(info) + if err != nil { + return nil, err + } + } else { + err := o.readMetaData() // reads info and headers, returning an error + if err != nil { + return nil, err + } + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// listFn is called from list to handle an object +type listFn func(remote string, object *api.File, isDirectory bool) error + +// errEndList is a sentinel used to end the list iteration now. +// listFn should return it to end the iteration with no errors. +var errEndList = errors.New("end list") + +// list lists the objects into the function supplied from +// the bucket and root supplied +// +// dir is the starting directory, "" for root +// +// level is the depth to search to +// +// If prefix is set then startFileName is used as a prefix which all +// files must have +// +// If limit is > 0 then it limits to that many files (must be less +// than 1000) +// +// If hidden is set then it will list the hidden (deleted) files too. +func (f *Fs) list(dir string, recurse bool, prefix string, limit int, hidden bool, fn listFn) error { + root := f.root + if dir != "" { + root += dir + "/" + } + delimiter := "" + if !recurse { + delimiter = "/" + } + bucketID, err := f.getBucketID() + if err != nil { + return err + } + chunkSize := 1000 + if limit > 0 { + chunkSize = limit + } + var request = api.ListFileNamesRequest{ + BucketID: bucketID, + MaxFileCount: chunkSize, + Prefix: root, + Delimiter: delimiter, + } + prefix = root + prefix + if prefix != "" { + request.StartFileName = prefix + } + opts := rest.Opts{ + Method: "POST", + Path: "/b2_list_file_names", + } + if hidden { + opts.Path = "/b2_list_file_versions" + } + for { + var response api.ListFileNamesResponse + err := f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(&opts, &request, &response) + return f.shouldRetry(resp, err) + }) + if err != nil { + return err + } + for i := range response.Files { + file := &response.Files[i] + // Finish if file name no longer has prefix + if prefix != "" && !strings.HasPrefix(file.Name, prefix) { + return nil + } + if !strings.HasPrefix(file.Name, f.root) { + fs.Debugf(f, "Odd name received %q", file.Name) + continue + } + remote := file.Name[len(f.root):] + // Check for directory + isDirectory := strings.HasSuffix(remote, "/") + if isDirectory { + remote = remote[:len(remote)-1] + } + // Send object + err = fn(remote, file, isDirectory) + if err != nil { + if err == errEndList { + return nil + } + return err + } + } + // end if no NextFileName + if response.NextFileName == nil { + break + } + request.StartFileName = *response.NextFileName + if response.NextFileID != nil { + request.StartFileID = *response.NextFileID + } + } + return nil +} + +// Convert a list item into a DirEntry +func (f *Fs) itemToDirEntry(remote string, object *api.File, isDirectory bool, last *string) (fs.DirEntry, error) { + if isDirectory { + d := fs.NewDir(remote, time.Time{}) + return d, nil + } + if remote == *last { + remote = object.UploadTimestamp.AddVersion(remote) + } else { + *last = remote + } + // hide objects represent deleted files which we don't list + if object.Action == "hide" { + return nil, nil + } + o, err := f.newObjectWithInfo(remote, object) + if err != nil { + return nil, err + } + return o, nil +} + +// mark the bucket as being OK +func (f *Fs) markBucketOK() { + if f.bucket != "" { + f.bucketOKMu.Lock() + f.bucketOK = true + f.bucketOKMu.Unlock() + } +} + +// listDir lists a single directory +func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { + last := "" + err = f.list(dir, false, "", 0, f.opt.Versions, func(remote string, object *api.File, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory, &last) + if err != nil { + return err + } + if entry != nil { + entries = append(entries, entry) + } + return nil + }) + if err != nil { + return nil, err + } + // bucket must be present if listing succeeded + f.markBucketOK() + return entries, nil +} + +// listBuckets returns all the buckets to out +func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) { + if dir != "" { + return nil, fs.ErrorListBucketRequired + } + err = f.listBucketsToFn(func(bucket *api.Bucket) error { + d := fs.NewDir(bucket.Name, time.Time{}) + entries = append(entries, d) + return nil + }) + if err != nil { + return nil, err + } + return entries, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + if f.bucket == "" { + return f.listBuckets(dir) + } + return f.listDir(dir) +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + if f.bucket == "" { + return fs.ErrorListBucketRequired + } + list := walk.NewListRHelper(callback) + last := "" + err = f.list(dir, true, "", 0, f.opt.Versions, func(remote string, object *api.File, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory, &last) + if err != nil { + return err + } + return list.Add(entry) + }) + if err != nil { + return err + } + // bucket must be present if listing succeeded + f.markBucketOK() + return list.Flush() +} + +// listBucketFn is called from listBucketsToFn to handle a bucket +type listBucketFn func(*api.Bucket) error + +// listBucketsToFn lists the buckets to the function supplied +func (f *Fs) listBucketsToFn(fn listBucketFn) error { + var account = api.ListBucketsRequest{ + AccountID: f.info.AccountID, + BucketID: f.info.Allowed.BucketID, + } + + var response api.ListBucketsResponse + opts := rest.Opts{ + Method: "POST", + Path: "/b2_list_buckets", + } + err := f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(&opts, &account, &response) + return f.shouldRetry(resp, err) + }) + if err != nil { + return err + } + for i := range response.Buckets { + err = fn(&response.Buckets[i]) + if err != nil { + return err + } + } + return nil +} + +// getBucketID finds the ID for the current bucket name +func (f *Fs) getBucketID() (bucketID string, err error) { + f.bucketIDMutex.Lock() + defer f.bucketIDMutex.Unlock() + if f._bucketID != "" { + return f._bucketID, nil + } + err = f.listBucketsToFn(func(bucket *api.Bucket) error { + if bucket.Name == f.bucket { + bucketID = bucket.ID + } + return nil + + }) + if bucketID == "" { + err = fs.ErrorDirNotFound + } + f._bucketID = bucketID + return bucketID, err +} + +// setBucketID sets the ID for the current bucket name +func (f *Fs) setBucketID(ID string) { + f.bucketIDMutex.Lock() + f._bucketID = ID + f.bucketIDMutex.Unlock() +} + +// clearBucketID clears the ID for the current bucket name +func (f *Fs) clearBucketID() { + f.bucketIDMutex.Lock() + f._bucketID = "" + f.bucketIDMutex.Unlock() +} + +// Put the object into the bucket +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + // Temporary Object under construction + fs := &Object{ + fs: f, + remote: src.Remote(), + } + return fs, fs.Update(in, src, options...) +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// Mkdir creates the bucket if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + f.bucketOKMu.Lock() + defer f.bucketOKMu.Unlock() + if f.bucketOK { + return nil + } + opts := rest.Opts{ + Method: "POST", + Path: "/b2_create_bucket", + } + var request = api.CreateBucketRequest{ + AccountID: f.info.AccountID, + Name: f.bucket, + Type: "allPrivate", + } + var response api.Bucket + err := f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(&opts, &request, &response) + return f.shouldRetry(resp, err) + }) + if err != nil { + if apiErr, ok := err.(*api.Error); ok { + if apiErr.Code == "duplicate_bucket_name" { + // Check this is our bucket - buckets are globally unique and this + // might be someone elses. + _, getBucketErr := f.getBucketID() + if getBucketErr == nil { + // found so it is our bucket + f.bucketOK = true + return nil + } + if getBucketErr != fs.ErrorDirNotFound { + fs.Debugf(f, "Error checking bucket exists: %v", getBucketErr) + } + } + } + return errors.Wrap(err, "failed to create bucket") + } + f.setBucketID(response.ID) + f.bucketOK = true + return nil +} + +// Rmdir deletes the bucket if the fs is at the root +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + f.bucketOKMu.Lock() + defer f.bucketOKMu.Unlock() + if f.root != "" || dir != "" { + return nil + } + opts := rest.Opts{ + Method: "POST", + Path: "/b2_delete_bucket", + } + bucketID, err := f.getBucketID() + if err != nil { + return err + } + var request = api.DeleteBucketRequest{ + ID: bucketID, + AccountID: f.info.AccountID, + } + var response api.Bucket + err = f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(&opts, &request, &response) + return f.shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "failed to delete bucket") + } + f.bucketOK = false + f.clearBucketID() + f.clearUploadURL() + return nil +} + +// Precision of the remote +func (f *Fs) Precision() time.Duration { + return time.Millisecond +} + +// hide hides a file on the remote +func (f *Fs) hide(Name string) error { + bucketID, err := f.getBucketID() + if err != nil { + return err + } + opts := rest.Opts{ + Method: "POST", + Path: "/b2_hide_file", + } + var request = api.HideFileRequest{ + BucketID: bucketID, + Name: Name, + } + var response api.File + err = f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(&opts, &request, &response) + return f.shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrapf(err, "failed to hide %q", Name) + } + return nil +} + +// deleteByID deletes a file version given Name and ID +func (f *Fs) deleteByID(ID, Name string) error { + opts := rest.Opts{ + Method: "POST", + Path: "/b2_delete_file_version", + } + var request = api.DeleteFileRequest{ + ID: ID, + Name: Name, + } + var response api.File + err := f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(&opts, &request, &response) + return f.shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrapf(err, "failed to delete %q", Name) + } + return nil +} + +// purge deletes all the files and directories +// +// if oldOnly is true then it deletes only non current files. +// +// Implemented here so we can make sure we delete old versions. +func (f *Fs) purge(oldOnly bool) error { + var errReturn error + var checkErrMutex sync.Mutex + var checkErr = func(err error) { + if err == nil { + return + } + checkErrMutex.Lock() + defer checkErrMutex.Unlock() + if errReturn == nil { + errReturn = err + } + } + + // Delete Config.Transfers in parallel + toBeDeleted := make(chan *api.File, fs.Config.Transfers) + var wg sync.WaitGroup + wg.Add(fs.Config.Transfers) + for i := 0; i < fs.Config.Transfers; i++ { + go func() { + defer wg.Done() + for object := range toBeDeleted { + accounting.Stats.Checking(object.Name) + checkErr(f.deleteByID(object.ID, object.Name)) + accounting.Stats.DoneChecking(object.Name) + } + }() + } + last := "" + checkErr(f.list("", true, "", 0, true, func(remote string, object *api.File, isDirectory bool) error { + if !isDirectory { + accounting.Stats.Checking(remote) + if oldOnly && last != remote { + if object.Action == "hide" { + fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID) + toBeDeleted <- object + } else { + fs.Debugf(remote, "Not deleting current version (id %q) %q", object.ID, object.Action) + } + } else { + fs.Debugf(remote, "Deleting (id %q)", object.ID) + toBeDeleted <- object + } + last = remote + accounting.Stats.DoneChecking(remote) + } + return nil + })) + close(toBeDeleted) + wg.Wait() + + if !oldOnly { + checkErr(f.Rmdir("")) + } + return errReturn +} + +// Purge deletes all the files and directories including the old versions. +func (f *Fs) Purge() error { + return f.purge(false) +} + +// CleanUp deletes all the hidden files. +func (f *Fs) CleanUp() error { + return f.purge(true) +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.SHA1) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the Sha-1 of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.SHA1 { + return "", hash.ErrUnsupported + } + if o.sha1 == "" { + // Error is logged in readMetaData + err := o.readMetaData() + if err != nil { + return "", err + } + } + return o.sha1, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.size +} + +// decodeMetaDataRaw sets the metadata from the data passed in +// +// Sets +// o.id +// o.modTime +// o.size +// o.sha1 +func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp api.Timestamp, Info map[string]string, mimeType string) (err error) { + o.id = ID + o.sha1 = SHA1 + o.mimeType = mimeType + // Read SHA1 from metadata if it exists and isn't set + if o.sha1 == "" || o.sha1 == "none" { + o.sha1 = Info[sha1Key] + } + o.size = Size + // Use the UploadTimestamp if can't get file info + o.modTime = time.Time(UploadTimestamp) + return o.parseTimeString(Info[timeKey]) +} + +// decodeMetaData sets the metadata in the object from an api.File +// +// Sets +// o.id +// o.modTime +// o.size +// o.sha1 +func (o *Object) decodeMetaData(info *api.File) (err error) { + return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType) +} + +// decodeMetaDataFileInfo sets the metadata in the object from an api.FileInfo +// +// Sets +// o.id +// o.modTime +// o.size +// o.sha1 +func (o *Object) decodeMetaDataFileInfo(info *api.FileInfo) (err error) { + return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType) +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// Sets +// o.id +// o.modTime +// o.size +// o.sha1 +func (o *Object) readMetaData() (err error) { + if o.id != "" { + return nil + } + maxSearched := 1 + var timestamp api.Timestamp + baseRemote := o.remote + if o.fs.opt.Versions { + timestamp, baseRemote = api.RemoveVersion(baseRemote) + maxSearched = maxVersions + } + var info *api.File + err = o.fs.list("", true, baseRemote, maxSearched, o.fs.opt.Versions, func(remote string, object *api.File, isDirectory bool) error { + if isDirectory { + return nil + } + if remote == baseRemote { + if !timestamp.IsZero() && !timestamp.Equal(object.UploadTimestamp) { + return nil + } + info = object + } + return errEndList // read only 1 item + }) + if err != nil { + if err == fs.ErrorDirNotFound { + return fs.ErrorObjectNotFound + } + return err + } + if info == nil { + return fs.ErrorObjectNotFound + } + return o.decodeMetaData(info) +} + +// timeString returns modTime as the number of milliseconds +// elapsed since January 1, 1970 UTC as a decimal string. +func timeString(modTime time.Time) string { + return strconv.FormatInt(modTime.UnixNano()/1E6, 10) +} + +// parseTimeString converts a decimal string number of milliseconds +// elapsed since January 1, 1970 UTC into a time.Time and stores it in +// the modTime variable. +func (o *Object) parseTimeString(timeString string) (err error) { + if timeString == "" { + return nil + } + unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64) + if err != nil { + fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err) + return err + } + o.modTime = time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC() + return nil +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +// +// SHA-1 will also be updated once the request has completed. +func (o *Object) ModTime() (result time.Time) { + // The error is logged in readMetaData + _ = o.readMetaData() + return o.modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + // Not possible with B2 + return fs.ErrorCantSetModTime +} + +// Storable returns if this object is storable +func (o *Object) Storable() bool { + return true +} + +// openFile represents an Object open for reading +type openFile struct { + o *Object // Object we are reading for + resp *http.Response // response of the GET + body io.Reader // reading from here + hash gohash.Hash // currently accumulating SHA1 + bytes int64 // number of bytes read on this connection + eof bool // whether we have read end of file +} + +// newOpenFile wraps an io.ReadCloser and checks the sha1sum +func newOpenFile(o *Object, resp *http.Response) *openFile { + file := &openFile{ + o: o, + resp: resp, + hash: sha1.New(), + } + file.body = io.TeeReader(resp.Body, file.hash) + return file +} + +// Read bytes from the object - see io.Reader +func (file *openFile) Read(p []byte) (n int, err error) { + n, err = file.body.Read(p) + file.bytes += int64(n) + if err == io.EOF { + file.eof = true + } + return +} + +// Close the object and checks the length and SHA1 if all the object +// was read +func (file *openFile) Close() (err error) { + // Close the body at the end + defer fs.CheckClose(file.resp.Body, &err) + + // If not end of file then can't check SHA1 + if !file.eof { + return nil + } + + // Check to see we read the correct number of bytes + if file.o.Size() != file.bytes { + return errors.Errorf("object corrupted on transfer - length mismatch (want %d got %d)", file.o.Size(), file.bytes) + } + + // Check the SHA1 + receivedSHA1 := file.o.sha1 + calculatedSHA1 := fmt.Sprintf("%x", file.hash.Sum(nil)) + if receivedSHA1 != "" && receivedSHA1 != calculatedSHA1 { + return errors.Errorf("object corrupted on transfer - SHA1 mismatch (want %q got %q)", receivedSHA1, calculatedSHA1) + } + + return nil +} + +// Check it satisfies the interfaces +var _ io.ReadCloser = &openFile{} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + opts := rest.Opts{ + Method: "GET", + RootURL: o.fs.info.DownloadURL, + Options: options, + } + // Download by id if set otherwise by name + if o.id != "" { + opts.Path += "/b2api/v1/b2_download_file_by_id?fileId=" + urlEncode(o.id) + } else { + opts.Path += "/file/" + urlEncode(o.fs.bucket) + "/" + urlEncode(o.fs.root+o.remote) + } + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return o.fs.shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to open for download") + } + + // Parse the time out of the headers if possible + err = o.parseTimeString(resp.Header.Get(timeHeader)) + if err != nil { + _ = resp.Body.Close() + return nil, err + } + // Read sha1 from header if it isn't set + if o.sha1 == "" { + o.sha1 = resp.Header.Get(sha1Header) + fs.Debugf(o, "Reading sha1 from header - %q", o.sha1) + // if sha1 header is "none" (in big files), then need + // to read it from the metadata + if o.sha1 == "none" { + o.sha1 = resp.Header.Get(sha1InfoHeader) + fs.Debugf(o, "Reading sha1 from info - %q", o.sha1) + } + } + // Don't check length or hash on partial content + if resp.StatusCode == http.StatusPartialContent { + return resp.Body, nil + } + return newOpenFile(o, resp), nil +} + +// dontEncode is the characters that do not need percent-encoding +// +// The characters that do not need percent-encoding are a subset of +// the printable ASCII characters: upper-case letters, lower-case +// letters, digits, ".", "_", "-", "/", "~", "!", "$", "'", "(", ")", +// "*", ";", "=", ":", and "@". All other byte values in a UTF-8 must +// be replaced with "%" and the two-digit hex value of the byte. +const dontEncode = (`abcdefghijklmnopqrstuvwxyz` + + `ABCDEFGHIJKLMNOPQRSTUVWXYZ` + + `0123456789` + + `._-/~!$'()*;=:@`) + +// noNeedToEncode is a bitmap of characters which don't need % encoding +var noNeedToEncode [256]bool + +func init() { + for _, c := range dontEncode { + noNeedToEncode[c] = true + } +} + +// urlEncode encodes in with % encoding +func urlEncode(in string) string { + var out bytes.Buffer + for i := 0; i < len(in); i++ { + c := in[i] + if noNeedToEncode[c] { + _ = out.WriteByte(c) + } else { + _, _ = out.WriteString(fmt.Sprintf("%%%2X", c)) + } + } + return out.String() +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + if o.fs.opt.Versions { + return errNotWithVersions + } + err = o.fs.Mkdir("") + if err != nil { + return err + } + size := src.Size() + + if size == -1 { + // Check if the file is large enough for a chunked upload (needs to be at least two chunks) + buf := o.fs.getUploadBlock() + n, err := io.ReadFull(in, buf) + if err == nil { + bufReader := bufio.NewReader(in) + in = bufReader + _, err = bufReader.Peek(1) + } + + if err == nil { + fs.Debugf(o, "File is big enough for chunked streaming") + up, err := o.fs.newLargeUpload(o, in, src) + if err != nil { + o.fs.putUploadBlock(buf) + return err + } + return up.Stream(buf) + } else if err == io.EOF || err == io.ErrUnexpectedEOF { + fs.Debugf(o, "File has %d bytes, which makes only one chunk. Using direct upload.", n) + defer o.fs.putUploadBlock(buf) + size = int64(n) + in = bytes.NewReader(buf[:n]) + } else { + return err + } + } else if size > int64(o.fs.opt.UploadCutoff) { + up, err := o.fs.newLargeUpload(o, in, src) + if err != nil { + return err + } + return up.Upload() + } + + modTime := src.ModTime() + + calculatedSha1, _ := src.Hash(hash.SHA1) + if calculatedSha1 == "" { + calculatedSha1 = "hex_digits_at_end" + har := newHashAppendingReader(in, sha1.New()) + size += int64(har.AdditionalLength()) + in = har + } + + // Get upload URL + upload, err := o.fs.getUploadURL() + if err != nil { + return err + } + defer func() { + // return it like this because we might nil it out + o.fs.returnUploadURL(upload) + }() + + // Headers for upload file + // + // Authorization + // required + // An upload authorization token, from b2_get_upload_url. + // + // X-Bz-File-Name + // required + // + // The name of the file, in percent-encoded UTF-8. See Files for requirements on file names. See String Encoding. + // + // Content-Type + // required + // + // The MIME type of the content of the file, which will be returned in + // the Content-Type header when downloading the file. Use the + // Content-Type b2/x-auto to automatically set the stored Content-Type + // post upload. In the case where a file extension is absent or the + // lookup fails, the Content-Type is set to application/octet-stream. The + // Content-Type mappings can be purused here. + // + // X-Bz-Content-Sha1 + // required + // + // The SHA1 checksum of the content of the file. B2 will check this when + // the file is uploaded, to make sure that the file arrived correctly. It + // will be returned in the X-Bz-Content-Sha1 header when the file is + // downloaded. + // + // X-Bz-Info-src_last_modified_millis + // optional + // + // If the original source of the file being uploaded has a last modified + // time concept, Backblaze recommends using this spelling of one of your + // ten X-Bz-Info-* headers (see below). Using a standard spelling allows + // different B2 clients and the B2 web user interface to interoperate + // correctly. The value should be a base 10 number which represents a UTC + // time when the original source file was last modified. It is a base 10 + // number of milliseconds since midnight, January 1, 1970 UTC. This fits + // in a 64 bit integer such as the type "long" in the programming + // language Java. It is intended to be compatible with Java's time + // long. For example, it can be passed directly into the Java call + // Date.setTime(long time). + // + // X-Bz-Info-* + // optional + // + // Up to 10 of these headers may be present. The * part of the header + // name is replace with the name of a custom field in the file + // information stored with the file, and the value is an arbitrary UTF-8 + // string, percent-encoded. The same info headers sent with the upload + // will be returned with the download. + + opts := rest.Opts{ + Method: "POST", + RootURL: upload.UploadURL, + Body: in, + ExtraHeaders: map[string]string{ + "Authorization": upload.AuthorizationToken, + "X-Bz-File-Name": urlEncode(o.fs.root + o.remote), + "Content-Type": fs.MimeType(src), + sha1Header: calculatedSha1, + timeHeader: timeString(modTime), + }, + ContentLength: &size, + } + // for go1.8 (see release notes) we must nil the Body if we want a + // "Content-Length: 0" header which b2 requires for all files. + if size == 0 { + opts.Body = nil + } + var response api.FileInfo + // Don't retry, return a retry error instead + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + resp, err := o.fs.srv.CallJSON(&opts, nil, &response) + retry, err := o.fs.shouldRetry(resp, err) + // On retryable error clear UploadURL + if retry { + fs.Debugf(o, "Clearing upload URL because of error: %v", err) + upload = nil + } + return retry, err + }) + if err != nil { + return err + } + return o.decodeMetaDataFileInfo(&response) +} + +// Remove an object +func (o *Object) Remove() error { + if o.fs.opt.Versions { + return errNotWithVersions + } + if o.fs.opt.HardDelete { + return o.fs.deleteByID(o.id, o.fs.root+o.remote) + } + return o.fs.hide(o.fs.root + o.remote) +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + return o.mimeType +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.id +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.Purger = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.CleanUpper = &Fs{} + _ fs.ListRer = &Fs{} + _ fs.Object = &Object{} + _ fs.MimeTyper = &Object{} + _ fs.IDer = &Object{} +) diff --git a/.rclone_repo/backend/b2/b2_internal_test.go b/.rclone_repo/backend/b2/b2_internal_test.go new file mode 100755 index 0000000..f64e83b --- /dev/null +++ b/.rclone_repo/backend/b2/b2_internal_test.go @@ -0,0 +1,170 @@ +package b2 + +import ( + "testing" + "time" + + "github.com/ncw/rclone/fstest" +) + +// Test b2 string encoding +// https://www.backblaze.com/b2/docs/string_encoding.html + +var encodeTest = []struct { + fullyEncoded string + minimallyEncoded string + plainText string +}{ + {fullyEncoded: "%20", minimallyEncoded: "+", plainText: " "}, + {fullyEncoded: "%21", minimallyEncoded: "!", plainText: "!"}, + {fullyEncoded: "%22", minimallyEncoded: "%22", plainText: "\""}, + {fullyEncoded: "%23", minimallyEncoded: "%23", plainText: "#"}, + {fullyEncoded: "%24", minimallyEncoded: "$", plainText: "$"}, + {fullyEncoded: "%25", minimallyEncoded: "%25", plainText: "%"}, + {fullyEncoded: "%26", minimallyEncoded: "%26", plainText: "&"}, + {fullyEncoded: "%27", minimallyEncoded: "'", plainText: "'"}, + {fullyEncoded: "%28", minimallyEncoded: "(", plainText: "("}, + {fullyEncoded: "%29", minimallyEncoded: ")", plainText: ")"}, + {fullyEncoded: "%2A", minimallyEncoded: "*", plainText: "*"}, + {fullyEncoded: "%2B", minimallyEncoded: "%2B", plainText: "+"}, + {fullyEncoded: "%2C", minimallyEncoded: "%2C", plainText: ","}, + {fullyEncoded: "%2D", minimallyEncoded: "-", plainText: "-"}, + {fullyEncoded: "%2E", minimallyEncoded: ".", plainText: "."}, + {fullyEncoded: "%2F", minimallyEncoded: "/", plainText: "/"}, + {fullyEncoded: "%30", minimallyEncoded: "0", plainText: "0"}, + {fullyEncoded: "%31", minimallyEncoded: "1", plainText: "1"}, + {fullyEncoded: "%32", minimallyEncoded: "2", plainText: "2"}, + {fullyEncoded: "%33", minimallyEncoded: "3", plainText: "3"}, + {fullyEncoded: "%34", minimallyEncoded: "4", plainText: "4"}, + {fullyEncoded: "%35", minimallyEncoded: "5", plainText: "5"}, + {fullyEncoded: "%36", minimallyEncoded: "6", plainText: "6"}, + {fullyEncoded: "%37", minimallyEncoded: "7", plainText: "7"}, + {fullyEncoded: "%38", minimallyEncoded: "8", plainText: "8"}, + {fullyEncoded: "%39", minimallyEncoded: "9", plainText: "9"}, + {fullyEncoded: "%3A", minimallyEncoded: ":", plainText: ":"}, + {fullyEncoded: "%3B", minimallyEncoded: ";", plainText: ";"}, + {fullyEncoded: "%3C", minimallyEncoded: "%3C", plainText: "<"}, + {fullyEncoded: "%3D", minimallyEncoded: "=", plainText: "="}, + {fullyEncoded: "%3E", minimallyEncoded: "%3E", plainText: ">"}, + {fullyEncoded: "%3F", minimallyEncoded: "%3F", plainText: "?"}, + {fullyEncoded: "%40", minimallyEncoded: "@", plainText: "@"}, + {fullyEncoded: "%41", minimallyEncoded: "A", plainText: "A"}, + {fullyEncoded: "%42", minimallyEncoded: "B", plainText: "B"}, + {fullyEncoded: "%43", minimallyEncoded: "C", plainText: "C"}, + {fullyEncoded: "%44", minimallyEncoded: "D", plainText: "D"}, + {fullyEncoded: "%45", minimallyEncoded: "E", plainText: "E"}, + {fullyEncoded: "%46", minimallyEncoded: "F", plainText: "F"}, + {fullyEncoded: "%47", minimallyEncoded: "G", plainText: "G"}, + {fullyEncoded: "%48", minimallyEncoded: "H", plainText: "H"}, + {fullyEncoded: "%49", minimallyEncoded: "I", plainText: "I"}, + {fullyEncoded: "%4A", minimallyEncoded: "J", plainText: "J"}, + {fullyEncoded: "%4B", minimallyEncoded: "K", plainText: "K"}, + {fullyEncoded: "%4C", minimallyEncoded: "L", plainText: "L"}, + {fullyEncoded: "%4D", minimallyEncoded: "M", plainText: "M"}, + {fullyEncoded: "%4E", minimallyEncoded: "N", plainText: "N"}, + {fullyEncoded: "%4F", minimallyEncoded: "O", plainText: "O"}, + {fullyEncoded: "%50", minimallyEncoded: "P", plainText: "P"}, + {fullyEncoded: "%51", minimallyEncoded: "Q", plainText: "Q"}, + {fullyEncoded: "%52", minimallyEncoded: "R", plainText: "R"}, + {fullyEncoded: "%53", minimallyEncoded: "S", plainText: "S"}, + {fullyEncoded: "%54", minimallyEncoded: "T", plainText: "T"}, + {fullyEncoded: "%55", minimallyEncoded: "U", plainText: "U"}, + {fullyEncoded: "%56", minimallyEncoded: "V", plainText: "V"}, + {fullyEncoded: "%57", minimallyEncoded: "W", plainText: "W"}, + {fullyEncoded: "%58", minimallyEncoded: "X", plainText: "X"}, + {fullyEncoded: "%59", minimallyEncoded: "Y", plainText: "Y"}, + {fullyEncoded: "%5A", minimallyEncoded: "Z", plainText: "Z"}, + {fullyEncoded: "%5B", minimallyEncoded: "%5B", plainText: "["}, + {fullyEncoded: "%5C", minimallyEncoded: "%5C", plainText: "\\"}, + {fullyEncoded: "%5D", minimallyEncoded: "%5D", plainText: "]"}, + {fullyEncoded: "%5E", minimallyEncoded: "%5E", plainText: "^"}, + {fullyEncoded: "%5F", minimallyEncoded: "_", plainText: "_"}, + {fullyEncoded: "%60", minimallyEncoded: "%60", plainText: "`"}, + {fullyEncoded: "%61", minimallyEncoded: "a", plainText: "a"}, + {fullyEncoded: "%62", minimallyEncoded: "b", plainText: "b"}, + {fullyEncoded: "%63", minimallyEncoded: "c", plainText: "c"}, + {fullyEncoded: "%64", minimallyEncoded: "d", plainText: "d"}, + {fullyEncoded: "%65", minimallyEncoded: "e", plainText: "e"}, + {fullyEncoded: "%66", minimallyEncoded: "f", plainText: "f"}, + {fullyEncoded: "%67", minimallyEncoded: "g", plainText: "g"}, + {fullyEncoded: "%68", minimallyEncoded: "h", plainText: "h"}, + {fullyEncoded: "%69", minimallyEncoded: "i", plainText: "i"}, + {fullyEncoded: "%6A", minimallyEncoded: "j", plainText: "j"}, + {fullyEncoded: "%6B", minimallyEncoded: "k", plainText: "k"}, + {fullyEncoded: "%6C", minimallyEncoded: "l", plainText: "l"}, + {fullyEncoded: "%6D", minimallyEncoded: "m", plainText: "m"}, + {fullyEncoded: "%6E", minimallyEncoded: "n", plainText: "n"}, + {fullyEncoded: "%6F", minimallyEncoded: "o", plainText: "o"}, + {fullyEncoded: "%70", minimallyEncoded: "p", plainText: "p"}, + {fullyEncoded: "%71", minimallyEncoded: "q", plainText: "q"}, + {fullyEncoded: "%72", minimallyEncoded: "r", plainText: "r"}, + {fullyEncoded: "%73", minimallyEncoded: "s", plainText: "s"}, + {fullyEncoded: "%74", minimallyEncoded: "t", plainText: "t"}, + {fullyEncoded: "%75", minimallyEncoded: "u", plainText: "u"}, + {fullyEncoded: "%76", minimallyEncoded: "v", plainText: "v"}, + {fullyEncoded: "%77", minimallyEncoded: "w", plainText: "w"}, + {fullyEncoded: "%78", minimallyEncoded: "x", plainText: "x"}, + {fullyEncoded: "%79", minimallyEncoded: "y", plainText: "y"}, + {fullyEncoded: "%7A", minimallyEncoded: "z", plainText: "z"}, + {fullyEncoded: "%7B", minimallyEncoded: "%7B", plainText: "{"}, + {fullyEncoded: "%7C", minimallyEncoded: "%7C", plainText: "|"}, + {fullyEncoded: "%7D", minimallyEncoded: "%7D", plainText: "}"}, + {fullyEncoded: "%7E", minimallyEncoded: "~", plainText: "~"}, + {fullyEncoded: "%7F", minimallyEncoded: "%7F", plainText: "\u007f"}, + {fullyEncoded: "%E8%87%AA%E7%94%B1", minimallyEncoded: "%E8%87%AA%E7%94%B1", plainText: "自由"}, + {fullyEncoded: "%F0%90%90%80", minimallyEncoded: "%F0%90%90%80", plainText: "𐐀"}, +} + +func TestUrlEncode(t *testing.T) { + for _, test := range encodeTest { + got := urlEncode(test.plainText) + if got != test.minimallyEncoded && got != test.fullyEncoded { + t.Errorf("urlEncode(%q) got %q wanted %q or %q", test.plainText, got, test.minimallyEncoded, test.fullyEncoded) + } + } +} + +func TestTimeString(t *testing.T) { + for _, test := range []struct { + in time.Time + want string + }{ + {fstest.Time("1970-01-01T00:00:00.000000000Z"), "0"}, + {fstest.Time("2001-02-03T04:05:10.123123123Z"), "981173110123"}, + {fstest.Time("2001-02-03T05:05:10.123123123+01:00"), "981173110123"}, + } { + got := timeString(test.in) + if test.want != got { + t.Logf("%v: want %v got %v", test.in, test.want, got) + } + } + +} + +func TestParseTimeString(t *testing.T) { + for _, test := range []struct { + in string + want time.Time + wantError string + }{ + {"0", fstest.Time("1970-01-01T00:00:00.000000000Z"), ""}, + {"981173110123", fstest.Time("2001-02-03T04:05:10.123000000Z"), ""}, + {"", time.Time{}, ""}, + {"potato", time.Time{}, `strconv.ParseInt: parsing "potato": invalid syntax`}, + } { + o := Object{} + err := o.parseTimeString(test.in) + got := o.modTime + var gotError string + if err != nil { + gotError = err.Error() + } + if test.want != got { + t.Logf("%v: want %v got %v", test.in, test.want, got) + } + if test.wantError != gotError { + t.Logf("%v: want error %v got error %v", test.in, test.wantError, gotError) + } + } + +} diff --git a/.rclone_repo/backend/b2/b2_test.go b/.rclone_repo/backend/b2/b2_test.go new file mode 100755 index 0000000..b51d68e --- /dev/null +++ b/.rclone_repo/backend/b2/b2_test.go @@ -0,0 +1,17 @@ +// Test B2 filesystem interface +package b2_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/b2" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestB2:", + NilObject: (*b2.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/b2/upload.go b/.rclone_repo/backend/b2/upload.go new file mode 100755 index 0000000..76b2f32 --- /dev/null +++ b/.rclone_repo/backend/b2/upload.go @@ -0,0 +1,433 @@ +// Upload large files for b2 +// +// Docs - https://www.backblaze.com/b2/docs/large_files.html + +package b2 + +import ( + "bytes" + "crypto/sha1" + "encoding/hex" + "fmt" + gohash "hash" + "io" + "strings" + "sync" + + "github.com/ncw/rclone/backend/b2/api" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/accounting" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" +) + +type hashAppendingReader struct { + h gohash.Hash + in io.Reader + hexSum string + hexReader io.Reader +} + +// Read returns bytes all bytes from the original reader, then the hex sum +// of what was read so far, then EOF. +func (har *hashAppendingReader) Read(b []byte) (int, error) { + if har.hexReader == nil { + n, err := har.in.Read(b) + if err == io.EOF { + har.in = nil // allow GC + err = nil // allow reading hexSum before EOF + + har.hexSum = hex.EncodeToString(har.h.Sum(nil)) + har.hexReader = strings.NewReader(har.hexSum) + } + return n, err + } + return har.hexReader.Read(b) +} + +// AdditionalLength returns how many bytes the appended hex sum will take up. +func (har *hashAppendingReader) AdditionalLength() int { + return hex.EncodedLen(har.h.Size()) +} + +// HexSum returns the hash sum as hex. It's only available after the original +// reader has EOF'd. It's an empty string before that. +func (har *hashAppendingReader) HexSum() string { + return har.hexSum +} + +// newHashAppendingReader takes a Reader and a Hash and will append the hex sum +// after the original reader reaches EOF. The increased size depends on the +// given hash, which may be queried through AdditionalLength() +func newHashAppendingReader(in io.Reader, h gohash.Hash) *hashAppendingReader { + withHash := io.TeeReader(in, h) + return &hashAppendingReader{h: h, in: withHash} +} + +// largeUpload is used to control the upload of large files which need chunking +type largeUpload struct { + f *Fs // parent Fs + o *Object // object being uploaded + in io.Reader // read the data from here + wrap accounting.WrapFn // account parts being transferred + id string // ID of the file being uploaded + size int64 // total size + parts int64 // calculated number of parts, if known + sha1s []string // slice of SHA1s for each part + uploadMu sync.Mutex // lock for upload variable + uploads []*api.GetUploadPartURLResponse // result of get upload URL calls +} + +// newLargeUpload starts an upload of object o from in with metadata in src +func (f *Fs) newLargeUpload(o *Object, in io.Reader, src fs.ObjectInfo) (up *largeUpload, err error) { + remote := o.remote + size := src.Size() + parts := int64(0) + sha1SliceSize := int64(maxParts) + if size == -1 { + fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize) + } else { + parts = size / int64(o.fs.opt.ChunkSize) + if size%int64(o.fs.opt.ChunkSize) != 0 { + parts++ + } + if parts > maxParts { + return nil, errors.Errorf("%q too big (%d bytes) makes too many parts %d > %d - increase --b2-chunk-size", remote, size, parts, maxParts) + } + sha1SliceSize = parts + } + + modTime := src.ModTime() + opts := rest.Opts{ + Method: "POST", + Path: "/b2_start_large_file", + } + bucketID, err := f.getBucketID() + if err != nil { + return nil, err + } + var request = api.StartLargeFileRequest{ + BucketID: bucketID, + Name: o.fs.root + remote, + ContentType: fs.MimeType(src), + Info: map[string]string{ + timeKey: timeString(modTime), + }, + } + // Set the SHA1 if known + if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" { + request.Info[sha1Key] = calculatedSha1 + } + var response api.StartLargeFileResponse + err = f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(&opts, &request, &response) + return f.shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + // unwrap the accounting from the input, we use wrap to put it + // back on after the buffering + in, wrap := accounting.UnWrap(in) + up = &largeUpload{ + f: f, + o: o, + in: in, + wrap: wrap, + id: response.ID, + size: size, + parts: parts, + sha1s: make([]string, sha1SliceSize), + } + return up, nil +} + +// getUploadURL returns the upload info with the UploadURL and the AuthorizationToken +// +// This should be returned with returnUploadURL when finished +func (up *largeUpload) getUploadURL() (upload *api.GetUploadPartURLResponse, err error) { + up.uploadMu.Lock() + defer up.uploadMu.Unlock() + if len(up.uploads) == 0 { + opts := rest.Opts{ + Method: "POST", + Path: "/b2_get_upload_part_url", + } + var request = api.GetUploadPartURLRequest{ + ID: up.id, + } + err := up.f.pacer.Call(func() (bool, error) { + resp, err := up.f.srv.CallJSON(&opts, &request, &upload) + return up.f.shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to get upload URL") + } + } else { + upload, up.uploads = up.uploads[0], up.uploads[1:] + } + return upload, nil +} + +// returnUploadURL returns the UploadURL to the cache +func (up *largeUpload) returnUploadURL(upload *api.GetUploadPartURLResponse) { + if upload == nil { + return + } + up.uploadMu.Lock() + up.uploads = append(up.uploads, upload) + up.uploadMu.Unlock() +} + +// clearUploadURL clears the current UploadURL and the AuthorizationToken +func (up *largeUpload) clearUploadURL() { + up.uploadMu.Lock() + up.uploads = nil + up.uploadMu.Unlock() +} + +// Transfer a chunk +func (up *largeUpload) transferChunk(part int64, body []byte) error { + err := up.f.pacer.Call(func() (bool, error) { + fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body)) + + // Get upload URL + upload, err := up.getUploadURL() + if err != nil { + return false, err + } + + in := newHashAppendingReader(bytes.NewReader(body), sha1.New()) + size := int64(len(body)) + int64(in.AdditionalLength()) + + // Authorization + // + // An upload authorization token, from b2_get_upload_part_url. + // + // X-Bz-Part-Number + // + // A number from 1 to 10000. The parts uploaded for one file + // must have contiguous numbers, starting with 1. + // + // Content-Length + // + // The number of bytes in the file being uploaded. Note that + // this header is required; you cannot leave it out and just + // use chunked encoding. The minimum size of every part but + // the last one is 100MB. + // + // X-Bz-Content-Sha1 + // + // The SHA1 checksum of the this part of the file. B2 will + // check this when the part is uploaded, to make sure that the + // data arrived correctly. The same SHA1 checksum must be + // passed to b2_finish_large_file. + opts := rest.Opts{ + Method: "POST", + RootURL: upload.UploadURL, + Body: up.wrap(in), + ExtraHeaders: map[string]string{ + "Authorization": upload.AuthorizationToken, + "X-Bz-Part-Number": fmt.Sprintf("%d", part), + sha1Header: "hex_digits_at_end", + }, + ContentLength: &size, + } + + var response api.UploadPartResponse + + resp, err := up.f.srv.CallJSON(&opts, nil, &response) + retry, err := up.f.shouldRetry(resp, err) + if err != nil { + fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", part, retry, err, err) + } + // On retryable error clear PartUploadURL + if retry { + fs.Debugf(up.o, "Clearing part upload URL because of error: %v", err) + upload = nil + } + up.returnUploadURL(upload) + up.sha1s[part-1] = in.HexSum() + return retry, err + }) + if err != nil { + fs.Debugf(up.o, "Error sending chunk %d: %v", part, err) + } else { + fs.Debugf(up.o, "Done sending chunk %d", part) + } + return err +} + +// finish closes off the large upload +func (up *largeUpload) finish() error { + fs.Debugf(up.o, "Finishing large file upload with %d parts", up.parts) + opts := rest.Opts{ + Method: "POST", + Path: "/b2_finish_large_file", + } + var request = api.FinishLargeFileRequest{ + ID: up.id, + SHA1s: up.sha1s, + } + var response api.FileInfo + err := up.f.pacer.Call(func() (bool, error) { + resp, err := up.f.srv.CallJSON(&opts, &request, &response) + return up.f.shouldRetry(resp, err) + }) + if err != nil { + return err + } + return up.o.decodeMetaDataFileInfo(&response) +} + +// cancel aborts the large upload +func (up *largeUpload) cancel() error { + opts := rest.Opts{ + Method: "POST", + Path: "/b2_cancel_large_file", + } + var request = api.CancelLargeFileRequest{ + ID: up.id, + } + var response api.CancelLargeFileResponse + err := up.f.pacer.Call(func() (bool, error) { + resp, err := up.f.srv.CallJSON(&opts, &request, &response) + return up.f.shouldRetry(resp, err) + }) + return err +} + +func (up *largeUpload) managedTransferChunk(wg *sync.WaitGroup, errs chan error, part int64, buf []byte) { + wg.Add(1) + go func(part int64, buf []byte) { + defer wg.Done() + defer up.f.putUploadBlock(buf) + err := up.transferChunk(part, buf) + if err != nil { + select { + case errs <- err: + default: + } + } + }(part, buf) +} + +func (up *largeUpload) finishOrCancelOnError(err error, errs chan error) error { + if err == nil { + select { + case err = <-errs: + default: + } + } + if err != nil { + fs.Debugf(up.o, "Cancelling large file upload due to error: %v", err) + cancelErr := up.cancel() + if cancelErr != nil { + fs.Errorf(up.o, "Failed to cancel large file upload: %v", cancelErr) + } + return err + } + return up.finish() +} + +// Stream uploads the chunks from the input, starting with a required initial +// chunk. Assumes the file size is unknown and will upload until the input +// reaches EOF. +func (up *largeUpload) Stream(initialUploadBlock []byte) (err error) { + fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id) + errs := make(chan error, 1) + hasMoreParts := true + var wg sync.WaitGroup + + // Transfer initial chunk + up.size = int64(len(initialUploadBlock)) + up.managedTransferChunk(&wg, errs, 1, initialUploadBlock) + +outer: + for part := int64(2); hasMoreParts; part++ { + // Check any errors + select { + case err = <-errs: + break outer + default: + } + + // Get a block of memory + buf := up.f.getUploadBlock() + + // Read the chunk + var n int + n, err = io.ReadFull(up.in, buf) + if err == io.ErrUnexpectedEOF { + fs.Debugf(up.o, "Read less than a full chunk, making this the last one.") + buf = buf[:n] + hasMoreParts = false + err = nil + } else if err == io.EOF { + fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.") + up.f.putUploadBlock(buf) + err = nil + break outer + } else if err != nil { + // other kinds of errors indicate failure + up.f.putUploadBlock(buf) + break outer + } + + // Keep stats up to date + up.parts = part + up.size += int64(n) + if part > maxParts { + err = errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts) + break outer + } + + // Transfer the chunk + up.managedTransferChunk(&wg, errs, part, buf) + } + wg.Wait() + up.sha1s = up.sha1s[:up.parts] + + return up.finishOrCancelOnError(err, errs) +} + +// Upload uploads the chunks from the input +func (up *largeUpload) Upload() error { + fs.Debugf(up.o, "Starting upload of large file in %d chunks (id %q)", up.parts, up.id) + remaining := up.size + errs := make(chan error, 1) + var wg sync.WaitGroup + var err error +outer: + for part := int64(1); part <= up.parts; part++ { + // Check any errors + select { + case err = <-errs: + break outer + default: + } + + reqSize := remaining + if reqSize >= int64(up.f.opt.ChunkSize) { + reqSize = int64(up.f.opt.ChunkSize) + } + + // Get a block of memory + buf := up.f.getUploadBlock()[:reqSize] + + // Read the chunk + _, err = io.ReadFull(up.in, buf) + if err != nil { + up.f.putUploadBlock(buf) + break outer + } + + // Transfer the chunk + up.managedTransferChunk(&wg, errs, part, buf) + remaining -= reqSize + } + wg.Wait() + + return up.finishOrCancelOnError(err, errs) +} diff --git a/.rclone_repo/backend/box/api/types.go b/.rclone_repo/backend/box/api/types.go new file mode 100755 index 0000000..e26f052 --- /dev/null +++ b/.rclone_repo/backend/box/api/types.go @@ -0,0 +1,192 @@ +// Package api has type definitions for box +// +// Converted from the API docs with help from https://mholt.github.io/json-to-go/ +package api + +import ( + "encoding/json" + "fmt" + "time" +) + +const ( + // 2017-05-03T07:26:10-07:00 + timeFormat = `"` + time.RFC3339 + `"` +) + +// Time represents represents date and time information for the +// box API, by using RFC3339 +type Time time.Time + +// MarshalJSON turns a Time into JSON (in UTC) +func (t *Time) MarshalJSON() (out []byte, err error) { + timeString := (*time.Time)(t).Format(timeFormat) + return []byte(timeString), nil +} + +// UnmarshalJSON turns JSON into a Time +func (t *Time) UnmarshalJSON(data []byte) error { + newT, err := time.Parse(timeFormat, string(data)) + if err != nil { + return err + } + *t = Time(newT) + return nil +} + +// Error is returned from box when things go wrong +type Error struct { + Type string `json:"type"` + Status int `json:"status"` + Code string `json:"code"` + ContextInfo json.RawMessage + HelpURL string `json:"help_url"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +// Error returns a string for the error and statistifes the error interface +func (e *Error) Error() string { + out := fmt.Sprintf("Error %q (%d)", e.Code, e.Status) + if e.Message != "" { + out += ": " + e.Message + } + if e.ContextInfo != nil { + out += fmt.Sprintf(" (%+v)", e.ContextInfo) + } + return out +} + +// Check Error statisfies the error interface +var _ error = (*Error)(nil) + +// ItemFields are the fields needed for FileInfo +var ItemFields = "type,id,sequence_id,etag,sha1,name,size,created_at,modified_at,content_created_at,content_modified_at,item_status" + +// Types of things in Item +const ( + ItemTypeFolder = "folder" + ItemTypeFile = "file" + ItemStatusActive = "active" + ItemStatusTrashed = "trashed" + ItemStatusDeleted = "deleted" +) + +// Item describes a folder or a file as returned by Get Folder Items and others +type Item struct { + Type string `json:"type"` + ID string `json:"id"` + SequenceID string `json:"sequence_id"` + Etag string `json:"etag"` + SHA1 string `json:"sha1"` + Name string `json:"name"` + Size float64 `json:"size"` // box returns this in xEyy format for very large numbers - see #2261 + CreatedAt Time `json:"created_at"` + ModifiedAt Time `json:"modified_at"` + ContentCreatedAt Time `json:"content_created_at"` + ContentModifiedAt Time `json:"content_modified_at"` + ItemStatus string `json:"item_status"` // active, trashed if the file has been moved to the trash, and deleted if the file has been permanently deleted +} + +// ModTime returns the modification time of the item +func (i *Item) ModTime() (t time.Time) { + t = time.Time(i.ContentModifiedAt) + if t.IsZero() { + t = time.Time(i.ModifiedAt) + } + return t +} + +// FolderItems is returned from the GetFolderItems call +type FolderItems struct { + TotalCount int `json:"total_count"` + Entries []Item `json:"entries"` + Offset int `json:"offset"` + Limit int `json:"limit"` + Order []struct { + By string `json:"by"` + Direction string `json:"direction"` + } `json:"order"` +} + +// Parent defined the ID of the parent directory +type Parent struct { + ID string `json:"id"` +} + +// CreateFolder is the request for Create Folder +type CreateFolder struct { + Name string `json:"name"` + Parent Parent `json:"parent"` +} + +// UploadFile is the request for Upload File +type UploadFile struct { + Name string `json:"name"` + Parent Parent `json:"parent"` + ContentCreatedAt Time `json:"content_created_at"` + ContentModifiedAt Time `json:"content_modified_at"` +} + +// UpdateFileModTime is used in Update File Info +type UpdateFileModTime struct { + ContentModifiedAt Time `json:"content_modified_at"` +} + +// UpdateFileMove is the request for Upload File to change name and parent +type UpdateFileMove struct { + Name string `json:"name"` + Parent Parent `json:"parent"` +} + +// CopyFile is the request for Copy File +type CopyFile struct { + Name string `json:"name"` + Parent Parent `json:"parent"` +} + +// UploadSessionRequest is uses in Create Upload Session +type UploadSessionRequest struct { + FolderID string `json:"folder_id,omitempty"` // don't pass for update + FileSize int64 `json:"file_size"` + FileName string `json:"file_name,omitempty"` // optional for update +} + +// UploadSessionResponse is returned from Create Upload Session +type UploadSessionResponse struct { + TotalParts int `json:"total_parts"` + PartSize int64 `json:"part_size"` + SessionEndpoints struct { + ListParts string `json:"list_parts"` + Commit string `json:"commit"` + UploadPart string `json:"upload_part"` + Status string `json:"status"` + Abort string `json:"abort"` + } `json:"session_endpoints"` + SessionExpiresAt Time `json:"session_expires_at"` + ID string `json:"id"` + Type string `json:"type"` + NumPartsProcessed int `json:"num_parts_processed"` +} + +// Part defines the return from upload part call which are passed to commit upload also +type Part struct { + PartID string `json:"part_id"` + Offset int64 `json:"offset"` + Size int64 `json:"size"` + Sha1 string `json:"sha1"` +} + +// UploadPartResponse is returned from the upload part call +type UploadPartResponse struct { + Part Part `json:"part"` +} + +// CommitUpload is used in the Commit Upload call +type CommitUpload struct { + Parts []Part `json:"parts"` + Attributes struct { + ContentCreatedAt Time `json:"content_created_at"` + ContentModifiedAt Time `json:"content_modified_at"` + } `json:"attributes"` +} diff --git a/.rclone_repo/backend/box/box.go b/.rclone_repo/backend/box/box.go new file mode 100755 index 0000000..9ee8c9f --- /dev/null +++ b/.rclone_repo/backend/box/box.go @@ -0,0 +1,1092 @@ +// Package box provides an interface to the Box +// object storage system. +package box + +// FIXME Box only supports file names of 255 characters or less. Names +// that will not be supported are those that contain non-printable +// ascii, / or \, names with trailing spaces, and the special names +// “.” and “..”. + +// FIXME box can copy a directory + +import ( + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "time" + + "github.com/ncw/rclone/backend/box/api" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/dircache" + "github.com/ncw/rclone/lib/oauthutil" + "github.com/ncw/rclone/lib/pacer" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" + "golang.org/x/oauth2" +) + +const ( + rcloneClientID = "d0374ba6pgmaguie02ge15sv1mllndho" + rcloneEncryptedClientSecret = "sYbJYm99WB8jzeaLPU0OPDMJKIkZvD2qOn3SyEMfiJr03RdtDt3xcZEIudRhbIDL" + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential + rootID = "0" // ID of root folder is always this + rootURL = "https://api.box.com/2.0" + uploadURL = "https://upload.box.com/api/2.0" + listChunks = 1000 // chunk size to read directory listings + minUploadCutoff = 50000000 // upload cutoff can be no lower than this + defaultUploadCutoff = 50 * 1024 * 1024 +) + +// Globals +var ( + // Description of how to auth for this app + oauthConfig = &oauth2.Config{ + Scopes: nil, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://app.box.com/api/oauth2/authorize", + TokenURL: "https://app.box.com/api/oauth2/token", + }, + ClientID: rcloneClientID, + ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), + RedirectURL: oauthutil.RedirectURL, + } +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "box", + Description: "Box", + NewFs: NewFs, + Config: func(name string, m configmap.Mapper) { + err := oauthutil.Config("box", name, m, oauthConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + }, + Options: []fs.Option{{ + Name: config.ConfigClientID, + Help: "Box App Client Id.\nLeave blank normally.", + }, { + Name: config.ConfigClientSecret, + Help: "Box App Client Secret\nLeave blank normally.", + }, { + Name: "upload_cutoff", + Help: "Cutoff for switching to multipart upload.", + Default: fs.SizeSuffix(defaultUploadCutoff), + Advanced: true, + }, { + Name: "commit_retries", + Help: "Max number of times to try committing a multipart file.", + Default: 100, + Advanced: true, + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` + CommitRetries int `config:"commit_retries"` +} + +// Fs represents a remote box +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + features *fs.Features // optional features + srv *rest.Client // the connection to the one drive server + dirCache *dircache.DirCache // Map of directory path to directory id + pacer *pacer.Pacer // pacer for API calls + tokenRenewer *oauthutil.Renew // renew the token on expiry + uploadToken *pacer.TokenDispenser // control concurrency +} + +// Object describes a box object +// +// Will definitely have info but maybe not meta +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + hasMetaData bool // whether info below has been set + size int64 // size of the object + modTime time.Time // modification time of the object + id string // ID of the object + sha1 string // SHA-1 of the object content +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("box root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// parsePath parses an box 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(resp *http.Response, err error) (bool, error) { + authRety := false + + if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { + authRety = true + fs.Debugf(nil, "Should retry: %v", err) + } + return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// substitute reserved characters for box +func replaceReservedChars(x string) string { + // Backslash for FULLWIDTH REVERSE SOLIDUS + return strings.Replace(x, "\\", "＼", -1) +} + +// restore reserved characters for box +func restoreReservedChars(x string) string { + // FULLWIDTH REVERSE SOLIDUS for Backslash + return strings.Replace(x, "＼", "\\", -1) +} + +// readMetaDataForPath reads the metadata from the path +func (f *Fs) readMetaDataForPath(path string) (info *api.Item, err error) { + // defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err) + leaf, directoryID, err := f.dirCache.FindRootAndPath(path, false) + if err != nil { + if err == fs.ErrorDirNotFound { + return nil, fs.ErrorObjectNotFound + } + return nil, err + } + + found, err := f.listAll(directoryID, false, true, func(item *api.Item) bool { + if item.Name == leaf { + info = item + return true + } + return false + }) + if err != nil { + return nil, err + } + if !found { + return nil, fs.ErrorObjectNotFound + } + return info, nil +} + +// errorHandler parses a non 2xx error response into an error +func errorHandler(resp *http.Response) error { + // Decode error response + errResponse := new(api.Error) + err := rest.DecodeJSON(resp, &errResponse) + if err != nil { + fs.Debugf(nil, "Couldn't decode error response: %v", err) + } + if errResponse.Code == "" { + errResponse.Code = resp.Status + } + if errResponse.Status == 0 { + errResponse.Status = resp.StatusCode + } + return errResponse +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + if opt.UploadCutoff < minUploadCutoff { + return nil, errors.Errorf("box: upload cutoff (%v) must be greater than equal to %v", opt.UploadCutoff, fs.SizeSuffix(minUploadCutoff)) + } + + root = parsePath(root) + oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig) + if err != nil { + log.Fatalf("Failed to configure Box: %v", err) + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + srv: rest.NewClient(oAuthClient).SetRoot(rootURL), + pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), + uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers), + } + f.features = (&fs.Features{ + CaseInsensitive: true, + CanHaveEmptyDirectories: true, + }).Fill(f) + f.srv.SetErrorHandler(errorHandler) + + // Renew the token in the background + f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { + _, err := f.readMetaDataForPath("") + return err + }) + + // Get rootID + f.dirCache = dircache.New(root, rootID, f) + + // Find the current root + err = f.dirCache.FindRoot(false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + newF := *f + newF.dirCache = dircache.New(newRoot, rootID, &newF) + newF.root = newRoot + // Make new Fs which is the parent + err = newF.dirCache.FindRoot(false) + if err != nil { + // No root so return old f + return f, nil + } + _, err := newF.newObjectWithInfo(remote, nil) + if err != nil { + if err == fs.ErrorObjectNotFound { + // File doesn't exist so return old f + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return &newF, fs.ErrorIsFile + } + return f, nil +} + +// rootSlash returns root with a slash on if it is empty, otherwise empty string +func (f *Fs) rootSlash() string { + if f.root == "" { + return f.root + } + return f.root + "/" +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *api.Item) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + // Set info + err = o.setMetaData(info) + } else { + err = o.readMetaData() // reads info and meta, returning an error + } + if err != nil { + return nil, err + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) { + // Find the leaf in pathID + found, err = f.listAll(pathID, true, false, func(item *api.Item) bool { + if item.Name == leaf { + pathIDOut = item.ID + return true + } + return false + }) + return pathIDOut, found, err +} + +// fieldsValue creates a url.Values with fields set to those in api.Item +func fieldsValue() url.Values { + values := url.Values{} + values.Set("fields", api.ItemFields) + return values +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) { + // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) + var resp *http.Response + var info *api.Item + opts := rest.Opts{ + Method: "POST", + Path: "/folders", + Parameters: fieldsValue(), + } + mkdir := api.CreateFolder{ + Name: replaceReservedChars(leaf), + Parent: api.Parent{ + ID: pathID, + }, + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, &mkdir, &info) + return shouldRetry(resp, err) + }) + if err != nil { + //fmt.Printf("...Error %v\n", err) + return "", err + } + // fmt.Printf("...Id %q\n", *info.Id) + return info.ID, nil +} + +// list the objects into the function supplied +// +// If directories is set it only sends directories +// User function to process a File item from listAll +// +// Should return true to finish processing +type listAllFn func(*api.Item) bool + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { + opts := rest.Opts{ + Method: "GET", + Path: "/folders/" + dirID + "/items", + Parameters: fieldsValue(), + } + opts.Parameters.Set("limit", strconv.Itoa(listChunks)) + offset := 0 +OUTER: + for { + opts.Parameters.Set("offset", strconv.Itoa(offset)) + + var result api.FolderItems + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &result) + return shouldRetry(resp, err) + }) + if err != nil { + return found, errors.Wrap(err, "couldn't list files") + } + for i := range result.Entries { + item := &result.Entries[i] + if item.Type == api.ItemTypeFolder { + if filesOnly { + continue + } + } else if item.Type == api.ItemTypeFile { + if directoriesOnly { + continue + } + } else { + fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type) + continue + } + if item.ItemStatus != api.ItemStatusActive { + continue + } + item.Name = restoreReservedChars(item.Name) + if fn(item) { + found = true + break OUTER + } + } + offset += result.Limit + if offset >= result.TotalCount { + break + } + } + return +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + err = f.dirCache.FindRoot(false) + if err != nil { + return nil, err + } + directoryID, err := f.dirCache.FindDir(dir, false) + if err != nil { + return nil, err + } + var iErr error + _, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { + remote := path.Join(dir, info.Name) + if info.Type == api.ItemTypeFolder { + // cache the directory ID for later lookups + f.dirCache.Put(remote, info.ID) + d := fs.NewDir(remote, info.ModTime()).SetID(info.ID) + // FIXME more info from dir? + entries = append(entries, d) + } else if info.Type == api.ItemTypeFile { + o, err := f.newObjectWithInfo(remote, info) + if err != nil { + iErr = err + return true + } + entries = append(entries, o) + } + return false + }) + if err != nil { + return nil, err + } + if iErr != nil { + return nil, iErr + } + return entries, nil +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Returns the object, leaf, directoryID and error +// +// Used to create new objects +func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { + // Create the directory for the object if it doesn't exist + leaf, directoryID, err = f.dirCache.FindRootAndPath(remote, true) + if err != nil { + return + } + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + } + return o, leaf, directoryID, nil +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil) + switch err { + case nil: + return exisitingObj, exisitingObj.Update(in, src, options...) + case fs.ErrorObjectNotFound: + // Not found so create it + return f.PutUnchecked(in, src) + default: + return nil, err + } +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// PutUnchecked the object into the container +// +// This will produce an error if the object already exists +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime() + + o, _, _, err := f.createObject(remote, modTime, size) + if err != nil { + return nil, err + } + return o, o.Update(in, src, options...) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + err := f.dirCache.FindRoot(true) + if err != nil { + return err + } + if dir != "" { + _, err = f.dirCache.FindDir(dir, true) + } + return err +} + +// deleteObject removes an object by ID +func (f *Fs) deleteObject(id string) error { + opts := rest.Opts{ + Method: "DELETE", + Path: "/files/" + id, + NoResponse: true, + } + return f.pacer.Call(func() (bool, error) { + resp, err := f.srv.Call(&opts) + return shouldRetry(resp, err) + }) +} + +// purgeCheck removes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(dir string, check bool) error { + root := path.Join(f.root, dir) + if root == "" { + return errors.New("can't purge root directory") + } + dc := f.dirCache + err := dc.FindRoot(false) + if err != nil { + return err + } + rootID, err := dc.FindDir(dir, false) + if err != nil { + return err + } + + opts := rest.Opts{ + Method: "DELETE", + Path: "/folders/" + rootID, + Parameters: url.Values{}, + NoResponse: true, + } + opts.Parameters.Set("recursive", strconv.FormatBool(!check)) + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "rmdir failed") + } + f.dirCache.FlushDir(dir) + if err != nil { + return err + } + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + return f.purgeCheck(dir, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + err := srcObj.readMetaData() + if err != nil { + return nil, err + } + + srcPath := srcObj.fs.rootSlash() + srcObj.remote + dstPath := f.rootSlash() + remote + if strings.ToLower(srcPath) == strings.ToLower(dstPath) { + return nil, errors.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + // Copy the object + opts := rest.Opts{ + Method: "POST", + Path: "/files/" + srcObj.id + "/copy", + Parameters: fieldsValue(), + } + replacedLeaf := replaceReservedChars(leaf) + copyFile := api.CopyFile{ + Name: replacedLeaf, + Parent: api.Parent{ + ID: directoryID, + }, + } + var resp *http.Response + var info *api.Item + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, ©File, &info) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + err = dstObj.setMetaData(info) + if err != nil { + return nil, err + } + return dstObj, nil +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + return f.purgeCheck("", false) +} + +// move a file or folder +func (f *Fs) move(endpoint, id, leaf, directoryID string) (info *api.Item, err error) { + // Move the object + opts := rest.Opts{ + Method: "PUT", + Path: endpoint + id, + Parameters: fieldsValue(), + } + move := api.UpdateFileMove{ + Name: replaceReservedChars(leaf), + Parent: api.Parent{ + ID: directoryID, + }, + } + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, &move, &info) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + return info, nil +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + // Do the move + info, err := f.move("/files/", srcObj.id, leaf, directoryID) + if err != nil { + return nil, err + } + + err = dstObj.setMetaData(info) + if err != nil { + return nil, err + } + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := path.Join(srcFs.root, srcRemote) + dstPath := path.Join(f.root, dstRemote) + + // Refuse to move to or from the root + if srcPath == "" || dstPath == "" { + fs.Debugf(src, "DirMove error: Can't move root") + return errors.New("can't move root directory") + } + + // find the root src directory + err := srcFs.dirCache.FindRoot(false) + if err != nil { + return err + } + + // find the root dst directory + if dstRemote != "" { + err = f.dirCache.FindRoot(true) + if err != nil { + return err + } + } else { + if f.dirCache.FoundRoot() { + return fs.ErrorDirExists + } + } + + // Find ID of dst parent, creating subdirs if necessary + var leaf, directoryID string + findPath := dstRemote + if dstRemote == "" { + findPath = f.root + } + leaf, directoryID, err = f.dirCache.FindPath(findPath, true) + if err != nil { + return err + } + + // Check destination does not exist + if dstRemote != "" { + _, err = f.dirCache.FindDir(dstRemote, false) + if err == fs.ErrorDirNotFound { + // OK + } else if err != nil { + return err + } else { + return fs.ErrorDirExists + } + } + + // Find ID of src + srcID, err := srcFs.dirCache.FindDir(srcRemote, false) + if err != nil { + return err + } + + // Do the move + _, err = f.move("/folders/", srcID, leaf, directoryID) + if err != nil { + return err + } + srcFs.dirCache.FlushDir(srcRemote) + return nil +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *Fs) DirCacheFlush() { + f.dirCache.ResetRoot() +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.SHA1) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// srvPath returns a path for use in server +func (o *Object) srvPath() string { + return replaceReservedChars(o.fs.rootSlash() + o.remote) +} + +// Hash returns the SHA-1 of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.SHA1 { + return "", hash.ErrUnsupported + } + return o.sha1, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return 0 + } + return o.size +} + +// setMetaData sets the metadata from info +func (o *Object) setMetaData(info *api.Item) (err error) { + if info.Type != api.ItemTypeFile { + return errors.Wrapf(fs.ErrorNotAFile, "%q is %q", o.remote, info.Type) + } + o.hasMetaData = true + o.size = int64(info.Size) + o.sha1 = info.SHA1 + o.modTime = info.ModTime() + o.id = info.ID + return nil +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData() (err error) { + if o.hasMetaData { + return nil + } + info, err := o.fs.readMetaDataForPath(o.remote) + if err != nil { + if apiErr, ok := err.(*api.Error); ok { + if apiErr.Code == "not_found" || apiErr.Code == "trashed" { + return fs.ErrorObjectNotFound + } + } + return err + } + return o.setMetaData(info) +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// setModTime sets the modification time of the local fs object +func (o *Object) setModTime(modTime time.Time) (*api.Item, error) { + opts := rest.Opts{ + Method: "PUT", + Path: "/files/" + o.id, + Parameters: fieldsValue(), + } + update := api.UpdateFileModTime{ + ContentModifiedAt: api.Time(modTime), + } + var info *api.Item + err := o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.CallJSON(&opts, &update, &info) + return shouldRetry(resp, err) + }) + return info, err +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + info, err := o.setModTime(modTime) + if err != nil { + return err + } + return o.setMetaData(info) +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + if o.id == "" { + return nil, errors.New("can't download - no id") + } + fs.FixRangeOption(options, o.size) + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + Path: "/files/" + o.id + "/content", + Options: options, + } + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + return resp.Body, err +} + +// upload does a single non-multipart upload +// +// This is recommended for less than 50 MB of content +func (o *Object) upload(in io.Reader, leaf, directoryID string, modTime time.Time) (err error) { + upload := api.UploadFile{ + Name: replaceReservedChars(leaf), + ContentModifiedAt: api.Time(modTime), + ContentCreatedAt: api.Time(modTime), + Parent: api.Parent{ + ID: directoryID, + }, + } + + var resp *http.Response + var result api.FolderItems + opts := rest.Opts{ + Method: "POST", + Body: in, + MultipartMetadataName: "attributes", + MultipartContentName: "contents", + MultipartFileName: upload.Name, + RootURL: uploadURL, + } + // If object has an ID then it is existing so create a new version + if o.id != "" { + opts.Path = "/files/" + o.id + "/content" + } else { + opts.Path = "/files/content" + } + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(&opts, &upload, &result) + return shouldRetry(resp, err) + }) + if err != nil { + return err + } + if result.TotalCount != 1 || len(result.Entries) != 1 { + return errors.Errorf("failed to upload %v - not sure why", o) + } + return o.setMetaData(&result.Entries[0]) +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// If existing is set then it updates the object rather than creating a new one +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + o.fs.tokenRenewer.Start() + defer o.fs.tokenRenewer.Stop() + + size := src.Size() + modTime := src.ModTime() + remote := o.Remote() + + // Create the directory for the object if it doesn't exist + leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(remote, true) + if err != nil { + return err + } + + // Upload with simple or multipart + if size <= int64(o.fs.opt.UploadCutoff) { + err = o.upload(in, leaf, directoryID, modTime) + } else { + err = o.uploadMultipart(in, leaf, directoryID, size, modTime) + } + return err +} + +// Remove an object +func (o *Object) Remove() error { + return o.fs.deleteObject(o.id) +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.id +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.PutStreamer = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.IDer = (*Object)(nil) +) diff --git a/.rclone_repo/backend/box/box_test.go b/.rclone_repo/backend/box/box_test.go new file mode 100755 index 0000000..8355e7c --- /dev/null +++ b/.rclone_repo/backend/box/box_test.go @@ -0,0 +1,17 @@ +// Test Box filesystem interface +package box_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/box" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestBox:", + NilObject: (*box.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/box/upload.go b/.rclone_repo/backend/box/upload.go new file mode 100755 index 0000000..a3133cb --- /dev/null +++ b/.rclone_repo/backend/box/upload.go @@ -0,0 +1,275 @@ +// multpart upload for box + +package box + +import ( + "bytes" + "crypto/sha1" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "sync" + "time" + + "github.com/ncw/rclone/backend/box/api" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/accounting" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" +) + +// createUploadSession creates an upload session for the object +func (o *Object) createUploadSession(leaf, directoryID string, size int64) (response *api.UploadSessionResponse, err error) { + opts := rest.Opts{ + Method: "POST", + Path: "/files/upload_sessions", + RootURL: uploadURL, + } + request := api.UploadSessionRequest{ + FileSize: size, + } + // If object has an ID then it is existing so create a new version + if o.id != "" { + opts.Path = "/files/" + o.id + "/upload_sessions" + } else { + opts.Path = "/files/upload_sessions" + request.FolderID = directoryID + request.FileName = replaceReservedChars(leaf) + } + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(&opts, &request, &response) + return shouldRetry(resp, err) + }) + return +} + +// sha1Digest produces a digest using sha1 as per RFC3230 +func sha1Digest(digest []byte) string { + return "sha=" + base64.StdEncoding.EncodeToString(digest) +} + +// uploadPart uploads a part in an upload session +func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn) (response *api.UploadPartResponse, err error) { + chunkSize := int64(len(chunk)) + sha1sum := sha1.Sum(chunk) + opts := rest.Opts{ + Method: "PUT", + Path: "/files/upload_sessions/" + SessionID, + RootURL: uploadURL, + ContentType: "application/octet-stream", + ContentLength: &chunkSize, + ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize), + ExtraHeaders: map[string]string{ + "Digest": sha1Digest(sha1sum[:]), + }, + } + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + opts.Body = wrap(bytes.NewReader(chunk)) + resp, err = o.fs.srv.CallJSON(&opts, nil, &response) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + return response, nil +} + +// commitUpload finishes an upload session +func (o *Object) commitUpload(SessionID string, parts []api.Part, modTime time.Time, sha1sum []byte) (result *api.FolderItems, err error) { + opts := rest.Opts{ + Method: "POST", + Path: "/files/upload_sessions/" + SessionID + "/commit", + RootURL: uploadURL, + ExtraHeaders: map[string]string{ + "Digest": sha1Digest(sha1sum), + }, + } + request := api.CommitUpload{ + Parts: parts, + } + request.Attributes.ContentModifiedAt = api.Time(modTime) + request.Attributes.ContentCreatedAt = api.Time(modTime) + var body []byte + var resp *http.Response + // For discussion of this value see: + // https://github.com/ncw/rclone/issues/2054 + maxTries := o.fs.opt.CommitRetries + const defaultDelay = 10 + var tries int +outer: + for tries = 0; tries < maxTries; tries++ { + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(&opts, &request, nil) + if err != nil { + return shouldRetry(resp, err) + } + body, err = rest.ReadBody(resp) + return shouldRetry(resp, err) + }) + delay := defaultDelay + why := "unknown" + if err != nil { + // Sometimes we get 400 Error with + // parts_mismatch immediately after uploading + // the last part. Ignore this error and wait. + if boxErr, ok := err.(*api.Error); ok && boxErr.Code == "parts_mismatch" { + why = err.Error() + } else { + return nil, err + } + } else { + switch resp.StatusCode { + case http.StatusOK, http.StatusCreated: + break outer + case http.StatusAccepted: + why = "not ready yet" + delayString := resp.Header.Get("Retry-After") + if delayString != "" { + delay, err = strconv.Atoi(delayString) + if err != nil { + fs.Debugf(o, "Couldn't decode Retry-After header %q: %v", delayString, err) + delay = defaultDelay + } + } + default: + return nil, errors.Errorf("unknown HTTP status return %q (%d)", resp.Status, resp.StatusCode) + } + } + fs.Debugf(o, "commit multipart upload failed %d/%d - trying again in %d seconds (%s)", tries+1, maxTries, delay, why) + time.Sleep(time.Duration(delay) * time.Second) + } + if tries >= maxTries { + return nil, errors.New("too many tries to commit multipart upload - increase --low-level-retries") + } + err = json.Unmarshal(body, &result) + if err != nil { + return nil, errors.Wrapf(err, "couldn't decode commit response: %q", body) + } + return result, nil +} + +// abortUpload cancels an upload session +func (o *Object) abortUpload(SessionID string) (err error) { + opts := rest.Opts{ + Method: "DELETE", + Path: "/files/upload_sessions/" + SessionID, + RootURL: uploadURL, + NoResponse: true, + } + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return shouldRetry(resp, err) + }) + return err +} + +// uploadMultipart uploads a file using multipart upload +func (o *Object) uploadMultipart(in io.Reader, leaf, directoryID string, size int64, modTime time.Time) (err error) { + // Create upload session + session, err := o.createUploadSession(leaf, directoryID, size) + if err != nil { + return errors.Wrap(err, "multipart upload create session failed") + } + chunkSize := session.PartSize + fs.Debugf(o, "Multipart upload session started for %d parts of size %v", session.TotalParts, fs.SizeSuffix(chunkSize)) + + // Cancel the session if something went wrong + defer func() { + if err != nil { + fs.Debugf(o, "Cancelling multipart upload: %v", err) + cancelErr := o.abortUpload(session.ID) + if cancelErr != nil { + fs.Logf(o, "Failed to cancel multipart upload: %v", err) + } + } + }() + + // unwrap the accounting from the input, we use wrap to put it + // back on after the buffering + in, wrap := accounting.UnWrap(in) + + // Upload the chunks + remaining := size + position := int64(0) + parts := make([]api.Part, session.TotalParts) + hash := sha1.New() + errs := make(chan error, 1) + var wg sync.WaitGroup +outer: + for part := 0; part < session.TotalParts; part++ { + // Check any errors + select { + case err = <-errs: + break outer + default: + } + + reqSize := remaining + if reqSize >= int64(chunkSize) { + reqSize = int64(chunkSize) + } + + // Make a block of memory + buf := make([]byte, reqSize) + + // Read the chunk + _, err = io.ReadFull(in, buf) + if err != nil { + err = errors.Wrap(err, "multipart upload failed to read source") + break outer + } + + // Make the global hash (must be done sequentially) + _, _ = hash.Write(buf) + + // Transfer the chunk + wg.Add(1) + o.fs.uploadToken.Get() + go func(part int, position int64) { + defer wg.Done() + defer o.fs.uploadToken.Put() + fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize)) + partResponse, err := o.uploadPart(session.ID, position, size, buf, wrap) + if err != nil { + err = errors.Wrap(err, "multipart upload failed to upload part") + select { + case errs <- err: + default: + } + return + } + parts[part] = partResponse.Part + }(part, position) + + // ready for next block + remaining -= chunkSize + position += chunkSize + } + wg.Wait() + if err == nil { + select { + case err = <-errs: + default: + } + } + if err != nil { + return err + } + + // Finalise the upload session + result, err := o.commitUpload(session.ID, parts, modTime, hash.Sum(nil)) + if err != nil { + return errors.Wrap(err, "multipart upload failed to finalize") + } + + if result.TotalCount != 1 || len(result.Entries) != 1 { + return errors.Errorf("multipart upload failed %v - not sure why", o) + } + return o.setMetaData(&result.Entries[0]) +} diff --git a/.rclone_repo/backend/cache/cache.go b/.rclone_repo/backend/cache/cache.go new file mode 100755 index 0000000..d7541c0 --- /dev/null +++ b/.rclone_repo/backend/cache/cache.go @@ -0,0 +1,1586 @@ +// +build !plan9 + +package cache + +import ( + "context" + "fmt" + "io" + "os" + "os/signal" + "path" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/ncw/rclone/backend/crypt" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/fs/rc" + "github.com/ncw/rclone/fs/walk" + "github.com/ncw/rclone/lib/atexit" + "github.com/pkg/errors" + "golang.org/x/time/rate" +) + +const ( + // DefCacheChunkSize is the default value for chunk size + DefCacheChunkSize = fs.SizeSuffix(5 * 1024 * 1024) + // DefCacheTotalChunkSize is the default value for the maximum size of stored chunks + DefCacheTotalChunkSize = fs.SizeSuffix(10 * 1024 * 1024 * 1024) + // DefCacheChunkCleanInterval is the interval at which chunks are cleaned + DefCacheChunkCleanInterval = fs.Duration(time.Minute) + // DefCacheInfoAge is the default value for object info age + DefCacheInfoAge = fs.Duration(6 * time.Hour) + // DefCacheReadRetries is the default value for read retries + DefCacheReadRetries = 10 + // DefCacheTotalWorkers is how many workers run in parallel to download chunks + DefCacheTotalWorkers = 4 + // DefCacheChunkNoMemory will enable or disable in-memory storage for chunks + DefCacheChunkNoMemory = false + // DefCacheRps limits the number of requests per second to the source FS + DefCacheRps = -1 + // DefCacheWrites will cache file data on writes through the cache + DefCacheWrites = false + // DefCacheTmpWaitTime says how long should files be stored in local cache before being uploaded + DefCacheTmpWaitTime = fs.Duration(15 * time.Second) + // DefCacheDbWaitTime defines how long the cache backend should wait for the DB to be available + DefCacheDbWaitTime = fs.Duration(1 * time.Second) +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "cache", + Description: "Cache a remote", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "remote", + Help: "Remote to cache.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).", + Required: true, + }, { + Name: "plex_url", + Help: "The URL of the Plex server", + }, { + Name: "plex_username", + Help: "The username of the Plex user", + }, { + Name: "plex_password", + Help: "The password of the Plex user", + IsPassword: true, + }, { + Name: "plex_token", + Help: "The plex token for authentication - auto set normally", + Hide: fs.OptionHideBoth, + Advanced: true, + }, { + Name: "chunk_size", + Help: "The size of a chunk. Lower value good for slow connections but can affect seamless reading.", + Default: DefCacheChunkSize, + Examples: []fs.OptionExample{{ + Value: "1m", + Help: "1MB", + }, { + Value: "5M", + Help: "5 MB", + }, { + Value: "10M", + Help: "10 MB", + }}, + }, { + Name: "info_age", + Help: "How much time should object info (file size, file hashes etc) be stored in cache.\nUse a very high value if you don't plan on changing the source FS from outside the cache.\nAccepted units are: \"s\", \"m\", \"h\".", + Default: DefCacheInfoAge, + Examples: []fs.OptionExample{{ + Value: "1h", + Help: "1 hour", + }, { + Value: "24h", + Help: "24 hours", + }, { + Value: "48h", + Help: "48 hours", + }}, + }, { + Name: "chunk_total_size", + Help: "The maximum size of stored chunks. When the storage grows beyond this size, the oldest chunks will be deleted.", + Default: DefCacheTotalChunkSize, + Examples: []fs.OptionExample{{ + Value: "500M", + Help: "500 MB", + }, { + Value: "1G", + Help: "1 GB", + }, { + Value: "10G", + Help: "10 GB", + }}, + }, { + Name: "db_path", + Default: filepath.Join(config.CacheDir, "cache-backend"), + Help: "Directory to cache DB", + Advanced: true, + }, { + Name: "chunk_path", + Default: filepath.Join(config.CacheDir, "cache-backend"), + Help: "Directory to cache chunk files", + Advanced: true, + }, { + Name: "db_purge", + Default: false, + Help: "Purge the cache DB before", + Hide: fs.OptionHideConfigurator, + Advanced: true, + }, { + Name: "chunk_clean_interval", + Default: DefCacheChunkCleanInterval, + Help: "Interval at which chunk cleanup runs", + Advanced: true, + }, { + Name: "read_retries", + Default: DefCacheReadRetries, + Help: "How many times to retry a read from a cache storage", + Advanced: true, + }, { + Name: "workers", + Default: DefCacheTotalWorkers, + Help: "How many workers should run in parallel to download chunks", + Advanced: true, + }, { + Name: "chunk_no_memory", + Default: DefCacheChunkNoMemory, + Help: "Disable the in-memory cache for storing chunks during streaming", + Advanced: true, + }, { + Name: "rps", + Default: int(DefCacheRps), + Help: "Limits the number of requests per second to the source FS. -1 disables the rate limiter", + Advanced: true, + }, { + Name: "writes", + Default: DefCacheWrites, + Help: "Will cache file data on writes through the FS", + Advanced: true, + }, { + Name: "tmp_upload_path", + Default: "", + Help: "Directory to keep temporary files until they are uploaded to the cloud storage", + Advanced: true, + }, { + Name: "tmp_wait_time", + Default: DefCacheTmpWaitTime, + Help: "How long should files be stored in local cache before being uploaded", + Advanced: true, + }, { + Name: "db_wait_time", + Default: DefCacheDbWaitTime, + Help: "How long to wait for the DB to be available - 0 is unlimited", + Advanced: true, + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + Remote string `config:"remote"` + PlexURL string `config:"plex_url"` + PlexUsername string `config:"plex_username"` + PlexPassword string `config:"plex_password"` + PlexToken string `config:"plex_token"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` + InfoAge fs.Duration `config:"info_age"` + ChunkTotalSize fs.SizeSuffix `config:"chunk_total_size"` + DbPath string `config:"db_path"` + ChunkPath string `config:"chunk_path"` + DbPurge bool `config:"db_purge"` + ChunkCleanInterval fs.Duration `config:"chunk_clean_interval"` + ReadRetries int `config:"read_retries"` + TotalWorkers int `config:"workers"` + ChunkNoMemory bool `config:"chunk_no_memory"` + Rps int `config:"rps"` + StoreWrites bool `config:"writes"` + TempWritePath string `config:"tmp_upload_path"` + TempWaitTime fs.Duration `config:"tmp_wait_time"` + DbWaitTime fs.Duration `config:"db_wait_time"` +} + +// Fs represents a wrapped fs.Fs +type Fs struct { + fs.Fs + wrapper fs.Fs + + name string + root string + opt Options // parsed options + features *fs.Features // optional features + cache *Persistent + tempFs fs.Fs + + lastChunkCleanup time.Time + cleanupMu sync.Mutex + rateLimiter *rate.Limiter + plexConnector *plexConnector + backgroundRunner *backgroundWriter + cleanupChan chan bool + parentsForgetFn []func(string, fs.EntryType) + notifiedRemotes map[string]bool + notifiedMu sync.Mutex + parentsForgetMu sync.Mutex +} + +// parseRootPath returns a cleaned root path and a nil error or "" and an error when the path is invalid +func parseRootPath(path string) (string, error) { + return strings.Trim(path, "/"), nil +} + +// NewFs constructs a Fs from the path, container:path +func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + if opt.ChunkTotalSize < opt.ChunkSize*fs.SizeSuffix(opt.TotalWorkers) { + return nil, errors.Errorf("don't set cache-total-chunk-size(%v) less than cache-chunk-size(%v) * cache-workers(%v)", + opt.ChunkTotalSize, opt.ChunkSize, opt.TotalWorkers) + } + + if strings.HasPrefix(opt.Remote, name+":") { + return nil, errors.New("can't point cache remote at itself - check the value of the remote setting") + } + + rpath, err := parseRootPath(rootPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to clean root path %q", rootPath) + } + + remotePath := path.Join(opt.Remote, rpath) + wrappedFs, wrapErr := fs.NewFs(remotePath) + if wrapErr != nil && wrapErr != fs.ErrorIsFile { + return nil, errors.Wrapf(wrapErr, "failed to make remote %q to wrap", remotePath) + } + var fsErr error + fs.Debugf(name, "wrapped %v:%v at root %v", wrappedFs.Name(), wrappedFs.Root(), rpath) + if wrapErr == fs.ErrorIsFile { + fsErr = fs.ErrorIsFile + rpath = cleanPath(path.Dir(rpath)) + } + // configure cache backend + if opt.DbPurge { + fs.Debugf(name, "Purging the DB") + } + f := &Fs{ + Fs: wrappedFs, + name: name, + root: rpath, + opt: *opt, + lastChunkCleanup: time.Now().Truncate(time.Hour * 24 * 30), + cleanupChan: make(chan bool, 1), + notifiedRemotes: make(map[string]bool), + } + f.rateLimiter = rate.NewLimiter(rate.Limit(float64(opt.Rps)), opt.TotalWorkers) + + f.plexConnector = &plexConnector{} + if opt.PlexURL != "" { + if opt.PlexToken != "" { + f.plexConnector, err = newPlexConnectorWithToken(f, opt.PlexURL, opt.PlexToken) + if err != nil { + return nil, errors.Wrapf(err, "failed to connect to the Plex API %v", opt.PlexURL) + } + } else { + if opt.PlexPassword != "" && opt.PlexUsername != "" { + decPass, err := obscure.Reveal(opt.PlexPassword) + if err != nil { + decPass = opt.PlexPassword + } + f.plexConnector, err = newPlexConnector(f, opt.PlexURL, opt.PlexUsername, decPass, func(token string) { + m.Set("plex_token", token) + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to connect to the Plex API %v", opt.PlexURL) + } + } + } + } + + dbPath := f.opt.DbPath + chunkPath := f.opt.ChunkPath + // if the dbPath is non default but the chunk path is default, we overwrite the last to follow the same one as dbPath + if dbPath != filepath.Join(config.CacheDir, "cache-backend") && + chunkPath == filepath.Join(config.CacheDir, "cache-backend") { + chunkPath = dbPath + } + if filepath.Ext(dbPath) != "" { + dbPath = filepath.Dir(dbPath) + } + if filepath.Ext(chunkPath) != "" { + chunkPath = filepath.Dir(chunkPath) + } + err = os.MkdirAll(dbPath, os.ModePerm) + if err != nil { + return nil, errors.Wrapf(err, "failed to create cache directory %v", dbPath) + } + err = os.MkdirAll(chunkPath, os.ModePerm) + if err != nil { + return nil, errors.Wrapf(err, "failed to create cache directory %v", chunkPath) + } + + dbPath = filepath.Join(dbPath, name+".db") + chunkPath = filepath.Join(chunkPath, name) + fs.Infof(name, "Cache DB path: %v", dbPath) + fs.Infof(name, "Cache chunk path: %v", chunkPath) + f.cache, err = GetPersistent(dbPath, chunkPath, &Features{ + PurgeDb: opt.DbPurge, + DbWaitTime: time.Duration(opt.DbWaitTime), + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to start cache db") + } + // Trap SIGINT and SIGTERM to close the DB handle gracefully + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGHUP) + atexit.Register(func() { + if opt.PlexURL != "" { + f.plexConnector.closeWebsocket() + } + f.StopBackgroundRunners() + }) + go func() { + for { + s := <-c + if s == syscall.SIGHUP { + fs.Infof(f, "Clearing cache from signal") + f.DirCacheFlush() + } + } + }() + + fs.Infof(name, "Chunk Memory: %v", !f.opt.ChunkNoMemory) + fs.Infof(name, "Chunk Size: %v", f.opt.ChunkSize) + fs.Infof(name, "Chunk Total Size: %v", f.opt.ChunkTotalSize) + fs.Infof(name, "Chunk Clean Interval: %v", f.opt.ChunkCleanInterval) + fs.Infof(name, "Workers: %v", f.opt.TotalWorkers) + fs.Infof(name, "File Age: %v", f.opt.InfoAge) + if !f.opt.StoreWrites { + fs.Infof(name, "Cache Writes: enabled") + } + + if f.opt.TempWritePath != "" { + err = os.MkdirAll(f.opt.TempWritePath, os.ModePerm) + if err != nil { + return nil, errors.Wrapf(err, "failed to create cache directory %v", f.opt.TempWritePath) + } + f.opt.TempWritePath = filepath.ToSlash(f.opt.TempWritePath) + f.tempFs, err = fs.NewFs(f.opt.TempWritePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to create temp fs: %v", err) + } + fs.Infof(name, "Upload Temp Rest Time: %v", f.opt.TempWaitTime) + fs.Infof(name, "Upload Temp FS: %v", f.opt.TempWritePath) + f.backgroundRunner, _ = initBackgroundUploader(f) + go f.backgroundRunner.run() + } + + go func() { + for { + time.Sleep(time.Duration(f.opt.ChunkCleanInterval)) + select { + case <-f.cleanupChan: + fs.Infof(f, "stopping cleanup") + return + default: + fs.Debugf(f, "starting cleanup") + f.CleanUpCache(false) + } + } + }() + + if doChangeNotify := wrappedFs.Features().ChangeNotify; doChangeNotify != nil { + doChangeNotify(f.receiveChangeNotify, time.Duration(f.opt.ChunkCleanInterval)) + } + + f.features = (&fs.Features{ + CanHaveEmptyDirectories: true, + DuplicateFiles: false, // storage doesn't permit this + }).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs) + // override only those features that use a temp fs and it doesn't support them + //f.features.ChangeNotify = f.ChangeNotify + if f.opt.TempWritePath != "" { + if f.tempFs.Features().Copy == nil { + f.features.Copy = nil + } + if f.tempFs.Features().Move == nil { + f.features.Move = nil + } + if f.tempFs.Features().Move == nil { + f.features.Move = nil + } + if f.tempFs.Features().DirMove == nil { + f.features.DirMove = nil + } + if f.tempFs.Features().MergeDirs == nil { + f.features.MergeDirs = nil + } + } + // even if the wrapped fs doesn't support it, we still want it + f.features.DirCacheFlush = f.DirCacheFlush + + rc.Add(rc.Call{ + Path: "cache/expire", + Fn: f.httpExpireRemote, + Title: "Purge a remote from cache", + Help: ` +Purge a remote from the cache backend. Supports either a directory or a file. +Params: + - remote = path to remote (required) + - withData = true/false to delete cached data (chunks) as well (optional) + +Eg + + rclone rc cache/expire remote=path/to/sub/folder/ + rclone rc cache/expire remote=/ withData=true +`, + }) + + rc.Add(rc.Call{ + Path: "cache/stats", + Fn: f.httpStats, + Title: "Get cache stats", + Help: ` +Show statistics for the cache remote. +`, + }) + + return f, fsErr +} + +func (f *Fs) httpStats(in rc.Params) (out rc.Params, err error) { + out = make(rc.Params) + m, err := f.Stats() + if err != nil { + return out, errors.Errorf("error while getting cache stats") + } + out["status"] = "ok" + out["stats"] = m + return out, nil +} + +func (f *Fs) httpExpireRemote(in rc.Params) (out rc.Params, err error) { + out = make(rc.Params) + remoteInt, ok := in["remote"] + if !ok { + return out, errors.Errorf("remote is needed") + } + remote := remoteInt.(string) + withData := false + _, ok = in["withData"] + if ok { + withData = true + } + + if cleanPath(remote) != "" { + // if it's wrapped by crypt we need to check what format we got + if cryptFs, yes := f.isWrappedByCrypt(); yes { + _, err := cryptFs.DecryptFileName(remote) + // if it failed to decrypt then it is a decrypted format and we need to encrypt it + if err != nil { + remote = cryptFs.EncryptFileName(remote) + } + // else it's an encrypted format and we can use it as it is + } + + if !f.cache.HasEntry(path.Join(f.Root(), remote)) { + return out, errors.Errorf("%s doesn't exist in cache", remote) + } + } + + co := NewObject(f, remote) + err = f.cache.GetObject(co) + if err != nil { // it could be a dir + cd := NewDirectory(f, remote) + err := f.cache.ExpireDir(cd) + if err != nil { + return out, errors.WithMessage(err, "error expiring directory") + } + // notify vfs too + f.notifyChangeUpstream(cd.Remote(), fs.EntryDirectory) + out["status"] = "ok" + out["message"] = fmt.Sprintf("cached directory cleared: %v", remote) + return out, nil + } + // expire the entry + err = f.cache.ExpireObject(co, withData) + if err != nil { + return out, errors.WithMessage(err, "error expiring file") + } + // notify vfs too + f.notifyChangeUpstream(co.Remote(), fs.EntryObject) + + out["status"] = "ok" + out["message"] = fmt.Sprintf("cached file cleared: %v", remote) + return out, nil +} + +// receiveChangeNotify is a wrapper to notifications sent from the wrapped FS about changed files +func (f *Fs) receiveChangeNotify(forgetPath string, entryType fs.EntryType) { + if crypt, yes := f.isWrappedByCrypt(); yes { + decryptedPath, err := crypt.DecryptFileName(forgetPath) + if err == nil { + fs.Infof(decryptedPath, "received cache expiry notification") + } else { + fs.Infof(forgetPath, "received cache expiry notification") + } + } else { + fs.Infof(forgetPath, "received cache expiry notification") + } + // notify upstreams too (vfs) + f.notifyChangeUpstream(forgetPath, entryType) + + var cd *Directory + if entryType == fs.EntryObject { + co := NewObject(f, forgetPath) + err := f.cache.GetObject(co) + if err != nil { + fs.Debugf(f, "got change notification for non cached entry %v", co) + } + err = f.cache.ExpireObject(co, true) + if err != nil { + fs.Debugf(forgetPath, "notify: error expiring '%v': %v", co, err) + } + cd = NewDirectory(f, cleanPath(path.Dir(co.Remote()))) + } else { + cd = NewDirectory(f, forgetPath) + } + // we expire the dir + err := f.cache.ExpireDir(cd) + if err != nil { + fs.Debugf(forgetPath, "notify: error expiring '%v': %v", cd, err) + } else { + fs.Debugf(forgetPath, "notify: expired '%v'", cd) + } + + f.notifiedMu.Lock() + defer f.notifiedMu.Unlock() + f.notifiedRemotes[forgetPath] = true + f.notifiedRemotes[cd.Remote()] = true +} + +// notifyChangeUpstreamIfNeeded will check if the wrapped remote doesn't notify on changes +// or if we use a temp fs +func (f *Fs) notifyChangeUpstreamIfNeeded(remote string, entryType fs.EntryType) { + if f.Fs.Features().ChangeNotify == nil || f.opt.TempWritePath != "" { + f.notifyChangeUpstream(remote, entryType) + } +} + +// notifyChangeUpstream will loop through all the upstreams and notify +// of the provided remote (should be only a dir) +func (f *Fs) notifyChangeUpstream(remote string, entryType fs.EntryType) { + f.parentsForgetMu.Lock() + defer f.parentsForgetMu.Unlock() + if len(f.parentsForgetFn) > 0 { + for _, fn := range f.parentsForgetFn { + fn(remote, entryType) + } + } +} + +// ChangeNotify can subsribe multiple callers +// this is coupled with the wrapped fs ChangeNotify (if it supports it) +// and also notifies other caches (i.e VFS) to clear out whenever something changes +func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool { + f.parentsForgetMu.Lock() + defer f.parentsForgetMu.Unlock() + fs.Debugf(f, "subscribing to ChangeNotify") + f.parentsForgetFn = append(f.parentsForgetFn, notifyFunc) + return make(chan bool) +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// String returns a description of the FS +func (f *Fs) String() string { + return fmt.Sprintf("Cache remote %s:%s", f.name, f.root) +} + +// ChunkSize returns the configured chunk size +func (f *Fs) ChunkSize() int64 { + return int64(f.opt.ChunkSize) +} + +// InfoAge returns the configured file age +func (f *Fs) InfoAge() time.Duration { + return time.Duration(f.opt.InfoAge) +} + +// TempUploadWaitTime returns the configured temp file upload wait time +func (f *Fs) TempUploadWaitTime() time.Duration { + return time.Duration(f.opt.TempWaitTime) +} + +// NewObject finds the Object at remote. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + var err error + + fs.Debugf(f, "new object '%s'", remote) + co := NewObject(f, remote) + // search for entry in cache and validate it + err = f.cache.GetObject(co) + if err != nil { + fs.Debugf(remote, "find: error: %v", err) + } else if time.Now().After(co.CacheTs.Add(time.Duration(f.opt.InfoAge))) { + fs.Debugf(co, "find: cold object: %+v", co) + } else { + fs.Debugf(co, "find: warm object: %v, expiring on: %v", co, co.CacheTs.Add(time.Duration(f.opt.InfoAge))) + return co, nil + } + + // search for entry in source or temp fs + var obj fs.Object + if f.opt.TempWritePath != "" { + obj, err = f.tempFs.NewObject(remote) + // not found in temp fs + if err != nil { + fs.Debugf(remote, "find: not found in local cache fs") + obj, err = f.Fs.NewObject(remote) + } else { + fs.Debugf(obj, "find: found in local cache fs") + } + } else { + obj, err = f.Fs.NewObject(remote) + } + + // not found in either fs + if err != nil { + fs.Debugf(obj, "find failed: not found in either local or remote fs") + return nil, err + } + + // cache the new entry + co = ObjectFromOriginal(f, obj).persist() + fs.Debugf(co, "find: cached object") + return co, nil +} + +// List the objects and directories in dir into entries +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + fs.Debugf(f, "list '%s'", dir) + cd := ShallowDirectory(f, dir) + + // search for cached dir entries and validate them + entries, err = f.cache.GetDirEntries(cd) + if err != nil { + fs.Debugf(dir, "list: error: %v", err) + } else if time.Now().After(cd.CacheTs.Add(time.Duration(f.opt.InfoAge))) { + fs.Debugf(dir, "list: cold listing: %v", cd.CacheTs) + } else if len(entries) == 0 { + // TODO: read empty dirs from source? + fs.Debugf(dir, "list: empty listing") + } else { + fs.Debugf(dir, "list: warm %v from cache for: %v, expiring on: %v", len(entries), cd.abs(), cd.CacheTs.Add(time.Duration(f.opt.InfoAge))) + fs.Debugf(dir, "list: cached entries: %v", entries) + return entries, nil + } + // FIXME need to clean existing cached listing + + // we first search any temporary files stored locally + var cachedEntries fs.DirEntries + if f.opt.TempWritePath != "" { + queuedEntries, err := f.cache.searchPendingUploadFromDir(cd.abs()) + if err != nil { + fs.Errorf(dir, "list: error getting pending uploads: %v", err) + } else { + fs.Debugf(dir, "list: read %v from temp fs", len(queuedEntries)) + fs.Debugf(dir, "list: temp fs entries: %v", queuedEntries) + + for _, queuedRemote := range queuedEntries { + queuedEntry, err := f.tempFs.NewObject(f.cleanRootFromPath(queuedRemote)) + if err != nil { + fs.Debugf(dir, "list: temp file not found in local fs: %v", err) + continue + } + co := ObjectFromOriginal(f, queuedEntry).persist() + fs.Debugf(co, "list: cached temp object") + cachedEntries = append(cachedEntries, co) + } + } + } + + // search from the source + entries, err = f.Fs.List(dir) + if err != nil { + return nil, err + } + fs.Debugf(dir, "list: read %v from source", len(entries)) + fs.Debugf(dir, "list: source entries: %v", entries) + + // and then iterate over the ones from source (temp Objects will override source ones) + var batchDirectories []*Directory + for _, entry := range entries { + switch o := entry.(type) { + case fs.Object: + // skip over temporary objects (might be uploading) + found := false + for _, t := range cachedEntries { + if t.Remote() == o.Remote() { + found = true + break + } + } + if found { + continue + } + co := ObjectFromOriginal(f, o).persist() + cachedEntries = append(cachedEntries, co) + fs.Debugf(dir, "list: cached object: %v", co) + case fs.Directory: + cdd := DirectoryFromOriginal(f, o) + // check if the dir isn't expired and add it in cache if it isn't + if cdd2, err := f.cache.GetDir(cdd.abs()); err != nil || time.Now().Before(cdd2.CacheTs.Add(time.Duration(f.opt.InfoAge))) { + batchDirectories = append(batchDirectories, cdd) + } + cachedEntries = append(cachedEntries, cdd) + default: + fs.Debugf(entry, "list: Unknown object type %T", entry) + } + } + err = f.cache.AddBatchDir(batchDirectories) + if err != nil { + fs.Errorf(dir, "list: error caching directories from listing %v", dir) + } else { + fs.Debugf(dir, "list: cached directories: %v", len(batchDirectories)) + } + + // cache dir meta + t := time.Now() + cd.CacheTs = &t + err = f.cache.AddDir(cd) + if err != nil { + fs.Errorf(cd, "list: save error: '%v'", err) + } else { + fs.Debugf(dir, "list: cached dir: '%v', cache ts: %v", cd.abs(), cd.CacheTs) + } + + return cachedEntries, nil +} + +func (f *Fs) recurse(dir string, list *walk.ListRHelper) error { + entries, err := f.List(dir) + if err != nil { + return err + } + + for i := 0; i < len(entries); i++ { + innerDir, ok := entries[i].(fs.Directory) + if ok { + err := f.recurse(innerDir.Remote(), list) + if err != nil { + return err + } + } + + err := list.Add(entries[i]) + if err != nil { + return err + } + } + + return nil +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + fs.Debugf(f, "list recursively from '%s'", dir) + + // we check if the source FS supports ListR + // if it does, we'll use that to get all the entries, cache them and return + do := f.Fs.Features().ListR + if do != nil { + return do(dir, func(entries fs.DirEntries) error { + // we got called back with a set of entries so let's cache them and call the original callback + for _, entry := range entries { + switch o := entry.(type) { + case fs.Object: + _ = f.cache.AddObject(ObjectFromOriginal(f, o)) + case fs.Directory: + _ = f.cache.AddDir(DirectoryFromOriginal(f, o)) + default: + return errors.Errorf("Unknown object type %T", entry) + } + } + + // call the original callback + return callback(entries) + }) + } + + // if we're here, we're gonna do a standard recursive traversal and cache everything + list := walk.NewListRHelper(callback) + err = f.recurse(dir, list) + if err != nil { + return err + } + + return list.Flush() +} + +// Mkdir makes the directory (container, bucket) +func (f *Fs) Mkdir(dir string) error { + fs.Debugf(f, "mkdir '%s'", dir) + err := f.Fs.Mkdir(dir) + if err != nil { + return err + } + fs.Debugf(dir, "mkdir: created dir in source fs") + + cd := NewDirectory(f, cleanPath(dir)) + err = f.cache.AddDir(cd) + if err != nil { + fs.Errorf(dir, "mkdir: add error: %v", err) + } else { + fs.Debugf(cd, "mkdir: added to cache") + } + // expire parent of new dir + parentCd := NewDirectory(f, cleanPath(path.Dir(dir))) + err = f.cache.ExpireDir(parentCd) + if err != nil { + fs.Errorf(parentCd, "mkdir: cache expire error: %v", err) + } else { + fs.Infof(parentCd, "mkdir: cache expired") + } + // advertise to ChangeNotify if wrapped doesn't do that + f.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory) + + return nil +} + +// Rmdir removes the directory (container, bucket) if empty +func (f *Fs) Rmdir(dir string) error { + fs.Debugf(f, "rmdir '%s'", dir) + + if f.opt.TempWritePath != "" { + // pause background uploads + f.backgroundRunner.pause() + defer f.backgroundRunner.play() + + // we check if the source exists on the remote and make the same move on it too if it does + // otherwise, we skip this step + _, err := f.UnWrap().List(dir) + if err == nil { + err := f.Fs.Rmdir(dir) + if err != nil { + return err + } + fs.Debugf(dir, "rmdir: removed dir in source fs") + } + + var queuedEntries []*Object + err = walk.Walk(f.tempFs, dir, true, -1, func(path string, entries fs.DirEntries, err error) error { + for _, o := range entries { + if oo, ok := o.(fs.Object); ok { + co := ObjectFromOriginal(f, oo) + queuedEntries = append(queuedEntries, co) + } + } + return nil + }) + if err != nil { + fs.Errorf(dir, "rmdir: error getting pending uploads: %v", err) + } else { + fs.Debugf(dir, "rmdir: read %v from temp fs", len(queuedEntries)) + fs.Debugf(dir, "rmdir: temp fs entries: %v", queuedEntries) + if len(queuedEntries) > 0 { + fs.Errorf(dir, "rmdir: temporary dir not empty: %v", queuedEntries) + return fs.ErrorDirectoryNotEmpty + } + } + } else { + err := f.Fs.Rmdir(dir) + if err != nil { + return err + } + fs.Debugf(dir, "rmdir: removed dir in source fs") + } + + // remove dir data + d := NewDirectory(f, dir) + err := f.cache.RemoveDir(d.abs()) + if err != nil { + fs.Errorf(dir, "rmdir: remove error: %v", err) + } else { + fs.Debugf(d, "rmdir: removed from cache") + } + // expire parent + parentCd := NewDirectory(f, cleanPath(path.Dir(dir))) + err = f.cache.ExpireDir(parentCd) + if err != nil { + fs.Errorf(dir, "rmdir: cache expire error: %v", err) + } else { + fs.Infof(parentCd, "rmdir: cache expired") + } + // advertise to ChangeNotify if wrapped doesn't do that + f.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory) + + return nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + fs.Debugf(f, "move dir '%s'/'%s' -> '%s'/'%s'", src.Root(), srcRemote, f.Root(), dstRemote) + + do := f.Fs.Features().DirMove + if do == nil { + return fs.ErrorCantDirMove + } + srcFs, ok := src.(*Fs) + if !ok { + fs.Errorf(srcFs, "can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + if srcFs.Fs.Name() != f.Fs.Name() { + fs.Errorf(srcFs, "can't move directory - not wrapping same remotes") + return fs.ErrorCantDirMove + } + + if f.opt.TempWritePath != "" { + // pause background uploads + f.backgroundRunner.pause() + defer f.backgroundRunner.play() + + _, errInWrap := srcFs.UnWrap().List(srcRemote) + _, errInTemp := f.tempFs.List(srcRemote) + // not found in either fs + if errInWrap != nil && errInTemp != nil { + return fs.ErrorDirNotFound + } + + // we check if the source exists on the remote and make the same move on it too if it does + // otherwise, we skip this step + if errInWrap == nil { + err := do(srcFs.UnWrap(), srcRemote, dstRemote) + if err != nil { + return err + } + fs.Debugf(srcRemote, "movedir: dir moved in the source fs") + } + // we need to check if the directory exists in the temp fs + // and skip the move if it doesn't + if errInTemp != nil { + goto cleanup + } + + var queuedEntries []*Object + err := walk.Walk(f.tempFs, srcRemote, true, -1, func(path string, entries fs.DirEntries, err error) error { + for _, o := range entries { + if oo, ok := o.(fs.Object); ok { + co := ObjectFromOriginal(f, oo) + queuedEntries = append(queuedEntries, co) + if co.tempFileStartedUpload() { + fs.Errorf(co, "can't move - upload has already started. need to finish that") + return fs.ErrorCantDirMove + } + } + } + return nil + }) + if err != nil { + return err + } + fs.Debugf(srcRemote, "dirmove: read %v from temp fs", len(queuedEntries)) + fs.Debugf(srcRemote, "dirmove: temp fs entries: %v", queuedEntries) + + do := f.tempFs.Features().DirMove + if do == nil { + fs.Errorf(srcRemote, "dirmove: can't move dir in temp fs") + return fs.ErrorCantDirMove + } + err = do(f.tempFs, srcRemote, dstRemote) + if err != nil { + return err + } + err = f.cache.ReconcileTempUploads(f) + if err != nil { + return err + } + } else { + err := do(srcFs.UnWrap(), srcRemote, dstRemote) + if err != nil { + return err + } + fs.Debugf(srcRemote, "movedir: dir moved in the source fs") + } +cleanup: + + // delete src dir from cache along with all chunks + srcDir := NewDirectory(srcFs, srcRemote) + err := f.cache.RemoveDir(srcDir.abs()) + if err != nil { + fs.Errorf(srcDir, "dirmove: remove error: %v", err) + } else { + fs.Debugf(srcDir, "dirmove: removed cached dir") + } + // expire src parent + srcParent := NewDirectory(f, cleanPath(path.Dir(srcRemote))) + err = f.cache.ExpireDir(srcParent) + if err != nil { + fs.Errorf(srcParent, "dirmove: cache expire error: %v", err) + } else { + fs.Debugf(srcParent, "dirmove: cache expired") + } + // advertise to ChangeNotify if wrapped doesn't do that + f.notifyChangeUpstreamIfNeeded(srcParent.Remote(), fs.EntryDirectory) + + // expire parent dir at the destination path + dstParent := NewDirectory(f, cleanPath(path.Dir(dstRemote))) + err = f.cache.ExpireDir(dstParent) + if err != nil { + fs.Errorf(dstParent, "dirmove: cache expire error: %v", err) + } else { + fs.Debugf(dstParent, "dirmove: cache expired") + } + // advertise to ChangeNotify if wrapped doesn't do that + f.notifyChangeUpstreamIfNeeded(dstParent.Remote(), fs.EntryDirectory) + // TODO: precache dst dir and save the chunks + + return nil +} + +// cacheReader will split the stream of a reader to be cached at the same time it is read by the original source +func (f *Fs) cacheReader(u io.Reader, src fs.ObjectInfo, originalRead func(inn io.Reader)) { + // create the pipe and tee reader + pr, pw := io.Pipe() + tr := io.TeeReader(u, pw) + + // create channel to synchronize + done := make(chan bool) + defer close(done) + + go func() { + // notify the cache reader that we're complete after the source FS finishes + defer func() { + _ = pw.Close() + }() + // process original reading + originalRead(tr) + // signal complete + done <- true + }() + + go func() { + var offset int64 + for { + chunk := make([]byte, f.opt.ChunkSize) + readSize, err := io.ReadFull(pr, chunk) + // we ignore 3 failures which are ok: + // 1. EOF - original reading finished and we got a full buffer too + // 2. ErrUnexpectedEOF - original reading finished and partial buffer + // 3. ErrClosedPipe - source remote reader was closed (usually means it reached the end) and we need to stop too + // if we have a different error: we're going to error out the original reading too and stop this + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF && err != io.ErrClosedPipe { + fs.Errorf(src, "error saving new data in cache. offset: %v, err: %v", offset, err) + _ = pr.CloseWithError(err) + break + } + // if we have some bytes we cache them + if readSize > 0 { + chunk = chunk[:readSize] + err2 := f.cache.AddChunk(cleanPath(path.Join(f.root, src.Remote())), chunk, offset) + if err2 != nil { + fs.Errorf(src, "error saving new data in cache '%v'", err2) + _ = pr.CloseWithError(err2) + break + } + offset += int64(readSize) + } + // stuff should be closed but let's be sure + if err == io.EOF || err == io.ErrUnexpectedEOF || err == io.ErrClosedPipe { + _ = pr.Close() + break + } + } + + // signal complete + done <- true + }() + + // wait until both are done + for c := 0; c < 2; c++ { + <-done + } +} + +type putFn func(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) + +// put in to the remote path +func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) { + var err error + var obj fs.Object + + // queue for upload and store in temp fs if configured + if f.opt.TempWritePath != "" { + // we need to clear the caches before a put through temp fs + parentCd := NewDirectory(f, cleanPath(path.Dir(src.Remote()))) + _ = f.cache.ExpireDir(parentCd) + f.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory) + + obj, err = f.tempFs.Put(in, src, options...) + if err != nil { + fs.Errorf(obj, "put: failed to upload in temp fs: %v", err) + return nil, err + } + fs.Infof(obj, "put: uploaded in temp fs") + err = f.cache.addPendingUpload(path.Join(f.Root(), src.Remote()), false) + if err != nil { + fs.Errorf(obj, "put: failed to queue for upload: %v", err) + return nil, err + } + fs.Infof(obj, "put: queued for upload") + // if cache writes is enabled write it first through cache + } else if f.opt.StoreWrites { + f.cacheReader(in, src, func(inn io.Reader) { + obj, err = put(inn, src, options...) + }) + if err == nil { + fs.Debugf(obj, "put: uploaded to remote fs and saved in cache") + } + // last option: save it directly in remote fs + } else { + obj, err = put(in, src, options...) + if err == nil { + fs.Debugf(obj, "put: uploaded to remote fs") + } + } + // validate and stop if errors are found + if err != nil { + fs.Errorf(src, "put: error uploading: %v", err) + return nil, err + } + + // cache the new file + cachedObj := ObjectFromOriginal(f, obj) + + // deleting cached chunks and info to be replaced with new ones + _ = f.cache.RemoveObject(cachedObj.abs()) + + cachedObj.persist() + fs.Debugf(cachedObj, "put: added to cache") + + // expire parent + parentCd := NewDirectory(f, cleanPath(path.Dir(cachedObj.Remote()))) + err = f.cache.ExpireDir(parentCd) + if err != nil { + fs.Errorf(cachedObj, "put: cache expire error: %v", err) + } else { + fs.Infof(parentCd, "put: cache expired") + } + // advertise to ChangeNotify + f.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory) + + return cachedObj, nil +} + +// Put in to the remote path with the modTime given of the given size +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + fs.Debugf(f, "put data at '%s'", src.Remote()) + return f.put(in, src, options, f.Fs.Put) +} + +// PutUnchecked uploads the object +func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + do := f.Fs.Features().PutUnchecked + if do == nil { + return nil, errors.New("can't PutUnchecked") + } + fs.Debugf(f, "put data unchecked in '%s'", src.Remote()) + return f.put(in, src, options, do) +} + +// PutStream uploads the object +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + do := f.Fs.Features().PutStream + if do == nil { + return nil, errors.New("can't PutStream") + } + fs.Debugf(f, "put data streaming in '%s'", src.Remote()) + return f.put(in, src, options, do) +} + +// Copy src to this remote using server side copy operations. +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + fs.Debugf(f, "copy obj '%s' -> '%s'", src, remote) + + do := f.Fs.Features().Copy + if do == nil { + fs.Errorf(src, "source remote (%v) doesn't support Copy", src.Fs()) + return nil, fs.ErrorCantCopy + } + // the source must be a cached object or we abort + srcObj, ok := src.(*Object) + if !ok { + fs.Errorf(srcObj, "can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + // both the source cache fs and this cache fs need to wrap the same remote + if srcObj.CacheFs.Fs.Name() != f.Fs.Name() { + fs.Errorf(srcObj, "can't copy - not wrapping same remotes") + return nil, fs.ErrorCantCopy + } + // refresh from source or abort + if err := srcObj.refreshFromSource(false); err != nil { + fs.Errorf(f, "can't copy %v - %v", src, err) + return nil, fs.ErrorCantCopy + } + + if srcObj.isTempFile() { + // we check if the feature is stil active + if f.opt.TempWritePath == "" { + fs.Errorf(srcObj, "can't copy - this is a local cached file but this feature is turned off this run") + return nil, fs.ErrorCantCopy + } + + do = srcObj.ParentFs.Features().Copy + if do == nil { + fs.Errorf(src, "parent remote (%v) doesn't support Copy", srcObj.ParentFs) + return nil, fs.ErrorCantCopy + } + } + + obj, err := do(srcObj.Object, remote) + if err != nil { + fs.Errorf(srcObj, "error moving in cache: %v", err) + return nil, err + } + fs.Debugf(obj, "copy: file copied") + + // persist new + co := ObjectFromOriginal(f, obj).persist() + fs.Debugf(co, "copy: added to cache") + // expire the destination path + parentCd := NewDirectory(f, cleanPath(path.Dir(co.Remote()))) + err = f.cache.ExpireDir(parentCd) + if err != nil { + fs.Errorf(parentCd, "copy: cache expire error: %v", err) + } else { + fs.Infof(parentCd, "copy: cache expired") + } + // advertise to ChangeNotify if wrapped doesn't do that + f.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory) + // expire src parent + srcParent := NewDirectory(f, cleanPath(path.Dir(src.Remote()))) + err = f.cache.ExpireDir(srcParent) + if err != nil { + fs.Errorf(srcParent, "copy: cache expire error: %v", err) + } else { + fs.Infof(srcParent, "copy: cache expired") + } + // advertise to ChangeNotify if wrapped doesn't do that + f.notifyChangeUpstreamIfNeeded(srcParent.Remote(), fs.EntryDirectory) + + return co, nil +} + +// Move src to this remote using server side move operations. +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + fs.Debugf(f, "moving obj '%s' -> %s", src, remote) + + // if source fs doesn't support move abort + do := f.Fs.Features().Move + if do == nil { + fs.Errorf(src, "source remote (%v) doesn't support Move", src.Fs()) + return nil, fs.ErrorCantMove + } + // the source must be a cached object or we abort + srcObj, ok := src.(*Object) + if !ok { + fs.Errorf(srcObj, "can't move - not same remote type") + return nil, fs.ErrorCantMove + } + // both the source cache fs and this cache fs need to wrap the same remote + if srcObj.CacheFs.Fs.Name() != f.Fs.Name() { + fs.Errorf(srcObj, "can't move - not wrapping same remote types") + return nil, fs.ErrorCantMove + } + // refresh from source or abort + if err := srcObj.refreshFromSource(false); err != nil { + fs.Errorf(f, "can't move %v - %v", src, err) + return nil, fs.ErrorCantMove + } + + // if this is a temp object then we perform the changes locally + if srcObj.isTempFile() { + // we check if the feature is stil active + if f.opt.TempWritePath == "" { + fs.Errorf(srcObj, "can't move - this is a local cached file but this feature is turned off this run") + return nil, fs.ErrorCantMove + } + // pause background uploads + f.backgroundRunner.pause() + defer f.backgroundRunner.play() + + // started uploads can't be moved until they complete + if srcObj.tempFileStartedUpload() { + fs.Errorf(srcObj, "can't move - upload has already started. need to finish that") + return nil, fs.ErrorCantMove + } + do = f.tempFs.Features().Move + + // we must also update the pending queue + err := f.cache.updatePendingUpload(srcObj.abs(), func(item *tempUploadInfo) error { + item.DestPath = path.Join(f.Root(), remote) + item.AddedOn = time.Now() + return nil + }) + if err != nil { + fs.Errorf(srcObj, "failed to rename queued file for upload: %v", err) + return nil, fs.ErrorCantMove + } + fs.Debugf(srcObj, "move: queued file moved to %v", remote) + } + + obj, err := do(srcObj.Object, remote) + if err != nil { + fs.Errorf(srcObj, "error moving: %v", err) + return nil, err + } + fs.Debugf(obj, "move: file moved") + + // remove old + err = f.cache.RemoveObject(srcObj.abs()) + if err != nil { + fs.Errorf(srcObj, "move: remove error: %v", err) + } else { + fs.Debugf(srcObj, "move: removed from cache") + } + // expire old parent + parentCd := NewDirectory(f, cleanPath(path.Dir(srcObj.Remote()))) + err = f.cache.ExpireDir(parentCd) + if err != nil { + fs.Errorf(parentCd, "move: parent cache expire error: %v", err) + } else { + fs.Infof(parentCd, "move: cache expired") + } + // advertise to ChangeNotify if wrapped doesn't do that + f.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory) + // persist new + cachedObj := ObjectFromOriginal(f, obj).persist() + fs.Debugf(cachedObj, "move: added to cache") + // expire new parent + parentCd = NewDirectory(f, cleanPath(path.Dir(cachedObj.Remote()))) + err = f.cache.ExpireDir(parentCd) + if err != nil { + fs.Errorf(parentCd, "move: expire error: %v", err) + } else { + fs.Infof(parentCd, "move: cache expired") + } + // advertise to ChangeNotify if wrapped doesn't do that + f.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory) + + return cachedObj, nil +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return f.Fs.Hashes() +} + +// Purge all files in the root and the root directory +func (f *Fs) Purge() error { + fs.Infof(f, "purging cache") + f.cache.Purge() + + do := f.Fs.Features().Purge + if do == nil { + return nil + } + + err := do() + if err != nil { + return err + } + + return nil +} + +// CleanUp the trash in the Fs +func (f *Fs) CleanUp() error { + f.CleanUpCache(false) + + do := f.Fs.Features().CleanUp + if do == nil { + return nil + } + + return do() +} + +// About gets quota information from the Fs +func (f *Fs) About() (*fs.Usage, error) { + do := f.Fs.Features().About + if do == nil { + return nil, errors.New("About not supported") + } + return do() +} + +// Stats returns stats about the cache storage +func (f *Fs) Stats() (map[string]map[string]interface{}, error) { + return f.cache.Stats() +} + +// openRateLimited will execute a closure under a rate limiter watch +func (f *Fs) openRateLimited(fn func() (io.ReadCloser, error)) (io.ReadCloser, error) { + var err error + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + start := time.Now() + + if err = f.rateLimiter.Wait(ctx); err != nil { + return nil, err + } + + elapsed := time.Since(start) + if elapsed > time.Second*2 { + fs.Debugf(f, "rate limited: %s", elapsed) + } + return fn() +} + +// CleanUpCache will cleanup only the cache data that is expired +func (f *Fs) CleanUpCache(ignoreLastTs bool) { + f.cleanupMu.Lock() + defer f.cleanupMu.Unlock() + + if ignoreLastTs || time.Now().After(f.lastChunkCleanup.Add(time.Duration(f.opt.ChunkCleanInterval))) { + f.cache.CleanChunksBySize(int64(f.opt.ChunkTotalSize)) + f.lastChunkCleanup = time.Now() + } +} + +// StopBackgroundRunners will signall all the runners to stop their work +// can be triggered from a terminate signal or from testing between runs +func (f *Fs) StopBackgroundRunners() { + f.cleanupChan <- false + if f.opt.TempWritePath != "" && f.backgroundRunner != nil && f.backgroundRunner.isRunning() { + f.backgroundRunner.close() + } + f.cache.Close() + fs.Debugf(f, "Services stopped") +} + +// UnWrap returns the Fs that this Fs is wrapping +func (f *Fs) UnWrap() fs.Fs { + return f.Fs +} + +// WrapFs returns the Fs that is wrapping this Fs +func (f *Fs) WrapFs() fs.Fs { + return f.wrapper +} + +// SetWrapper sets the Fs that is wrapping this Fs +func (f *Fs) SetWrapper(wrapper fs.Fs) { + f.wrapper = wrapper +} + +// isWrappedByCrypt checks if this is wrapped by a crypt remote +func (f *Fs) isWrappedByCrypt() (*crypt.Fs, bool) { + if f.wrapper == nil { + return nil, false + } + c, ok := f.wrapper.(*crypt.Fs) + return c, ok +} + +// cleanRootFromPath trims the root of the current fs from a path +func (f *Fs) cleanRootFromPath(p string) string { + if f.Root() != "" { + p = p[len(f.Root()):] // trim out root + if len(p) > 0 { // remove first separator + p = p[1:] + } + } + + return p +} + +func (f *Fs) isRootInPath(p string) bool { + if f.Root() == "" { + return true + } + return strings.HasPrefix(p, f.Root()+"/") +} + +// DirCacheFlush flushes the dir cache +func (f *Fs) DirCacheFlush() { + _ = f.cache.RemoveDir("") +} + +// GetBackgroundUploadChannel returns a channel that can be listened to for remote activities that happen +// in the background +func (f *Fs) GetBackgroundUploadChannel() chan BackgroundUploadState { + if f.opt.TempWritePath != "" { + return f.backgroundRunner.notifyCh + } + return nil +} + +func (f *Fs) isNotifiedRemote(remote string) bool { + f.notifiedMu.Lock() + defer f.notifiedMu.Unlock() + + n, ok := f.notifiedRemotes[remote] + if !ok || !n { + return false + } + + delete(f.notifiedRemotes, remote) + return n +} + +func cleanPath(p string) string { + p = path.Clean(p) + if p == "." || p == "/" { + p = "" + } + + return p +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.PutUncheckeder = (*Fs)(nil) + _ fs.PutStreamer = (*Fs)(nil) + _ fs.CleanUpper = (*Fs)(nil) + _ fs.UnWrapper = (*Fs)(nil) + _ fs.Wrapper = (*Fs)(nil) + _ fs.ListRer = (*Fs)(nil) + _ fs.ChangeNotifier = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) +) diff --git a/.rclone_repo/backend/cache/cache_internal_test.go b/.rclone_repo/backend/cache/cache_internal_test.go new file mode 100755 index 0000000..12461df --- /dev/null +++ b/.rclone_repo/backend/cache/cache_internal_test.go @@ -0,0 +1,1673 @@ +// +build !plan9 + +package cache_test + +import ( + "bytes" + "io" + "io/ioutil" + "log" + "math/rand" + "os" + "path" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/pkg/errors" + + "encoding/base64" + goflag "flag" + "fmt" + "runtime/debug" + + "encoding/json" + "net/http" + + "github.com/ncw/rclone/backend/cache" + "github.com/ncw/rclone/backend/crypt" + _ "github.com/ncw/rclone/backend/drive" + "github.com/ncw/rclone/backend/local" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/object" + "github.com/ncw/rclone/fs/rc" + "github.com/ncw/rclone/fs/rc/rcflags" + "github.com/ncw/rclone/fstest" + "github.com/ncw/rclone/vfs" + "github.com/ncw/rclone/vfs/vfsflags" + "github.com/stretchr/testify/require" +) + +const ( + // these 2 passwords are test random + cryptPassword1 = "3XcvMMdsV3d-HGAReTMdNH-5FcX5q32_lUeA" // oGJdUbQc7s8 + cryptPassword2 = "NlgTBEIe-qibA7v-FoMfuX6Cw8KlLai_aMvV" // mv4mZW572HM + cryptedTextBase64 = "UkNMT05FAAC320i2xIee0BiNyknSPBn+Qcw3q9FhIFp3tvq6qlqvbsno3PnxmEFeJG3jDBnR/wku2gHWeQ==" // one content + cryptedText2Base64 = "UkNMT05FAAATcQkVsgjBh8KafCKcr0wdTa1fMmV0U8hsCLGFoqcvxKVmvv7wx3Hf5EXxFcki2FFV4sdpmSrb9Q==" // updated content + cryptedText3Base64 = "UkNMT05FAAB/f7YtYKbPfmk9+OX/ffN3qG3OEdWT+z74kxCX9V/YZwJ4X2DN3HOnUC3gKQ4Gcoud5UtNvQ==" // test content + letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +) + +var ( + remoteName string + mountDir string + uploadDir string + useMount bool + runInstance *run + errNotSupported = errors.New("not supported") + decryptedToEncryptedRemotes = map[string]string{ + "one": "lm4u7jjt3c85bf56vjqgeenuno", + "second": "qvt1ochrkcfbptp5mu9ugb2l14", + "test": "jn4tegjtpqro30t3o11thb4b5s", + "test2": "qakvqnh8ttei89e0gc76crpql4", + "data.bin": "0q2847tfko6mhj3dag3r809qbc", + "ticw/data.bin": "5mv97b0ule6pht33srae5pice8/0q2847tfko6mhj3dag3r809qbc", + "tiuufo/test/one": "vi6u1olqhirqv14cd8qlej1mgo/jn4tegjtpqro30t3o11thb4b5s/lm4u7jjt3c85bf56vjqgeenuno", + "tiuufo/test/second": "vi6u1olqhirqv14cd8qlej1mgo/jn4tegjtpqro30t3o11thb4b5s/qvt1ochrkcfbptp5mu9ugb2l14", + "tiutfo/test/one": "legd371aa8ol36tjfklt347qnc/jn4tegjtpqro30t3o11thb4b5s/lm4u7jjt3c85bf56vjqgeenuno", + "tiutfo/second/one": "legd371aa8ol36tjfklt347qnc/qvt1ochrkcfbptp5mu9ugb2l14/lm4u7jjt3c85bf56vjqgeenuno", + "second/one": "qvt1ochrkcfbptp5mu9ugb2l14/lm4u7jjt3c85bf56vjqgeenuno", + "test/one": "jn4tegjtpqro30t3o11thb4b5s/lm4u7jjt3c85bf56vjqgeenuno", + "test/second": "jn4tegjtpqro30t3o11thb4b5s/qvt1ochrkcfbptp5mu9ugb2l14", + "one/test": "lm4u7jjt3c85bf56vjqgeenuno/jn4tegjtpqro30t3o11thb4b5s", + "one/test/data.bin": "lm4u7jjt3c85bf56vjqgeenuno/jn4tegjtpqro30t3o11thb4b5s/0q2847tfko6mhj3dag3r809qbc", + "second/test/data.bin": "qvt1ochrkcfbptp5mu9ugb2l14/jn4tegjtpqro30t3o11thb4b5s/0q2847tfko6mhj3dag3r809qbc", + "test/third": "jn4tegjtpqro30t3o11thb4b5s/2nd7fjiop5h3ihfj1vl953aa5g", + "test/0.bin": "jn4tegjtpqro30t3o11thb4b5s/e6frddt058b6kvbpmlstlndmtk", + "test/1.bin": "jn4tegjtpqro30t3o11thb4b5s/kck472nt1k7qbmob0mt1p1crgc", + "test/2.bin": "jn4tegjtpqro30t3o11thb4b5s/744oe9ven2rmak4u27if51qk24", + "test/3.bin": "jn4tegjtpqro30t3o11thb4b5s/2bjd8kef0u5lmsu6qhqll34bcs", + "test/4.bin": "jn4tegjtpqro30t3o11thb4b5s/cvjs73iv0a82v0c7r67avllh7s", + "test/5.bin": "jn4tegjtpqro30t3o11thb4b5s/0plkdo790b6bnmt33qsdqmhv9c", + "test/6.bin": "jn4tegjtpqro30t3o11thb4b5s/s5r633srnjtbh83893jovjt5d0", + "test/7.bin": "jn4tegjtpqro30t3o11thb4b5s/6rq45tr9bjsammku622flmqsu4", + "test/8.bin": "jn4tegjtpqro30t3o11thb4b5s/37bc6tcl3e31qb8cadvjb749vk", + "test/9.bin": "jn4tegjtpqro30t3o11thb4b5s/t4pr35hnls32789o8fk0chk1ec", + } +) + +func init() { + goflag.StringVar(&remoteName, "remote-internal", "TestInternalCache", "Remote to test with, defaults to local filesystem") + goflag.StringVar(&mountDir, "mount-dir-internal", "", "") + goflag.StringVar(&uploadDir, "upload-dir-internal", "", "") + goflag.BoolVar(&useMount, "cache-use-mount", false, "Test only with mount") +} + +// TestMain drives the tests +func TestMain(m *testing.M) { + goflag.Parse() + var rc int + + log.Printf("Running with the following params: \n remote: %v, \n mount: %v", remoteName, useMount) + runInstance = newRun() + rc = m.Run() + os.Exit(rc) +} + +func TestInternalListRootAndInnerRemotes(t *testing.T) { + id := fmt.Sprintf("tilrair%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + // Instantiate inner fs + innerFolder := "inner" + runInstance.mkdir(t, rootFs, innerFolder) + rootFs2, boltDb2 := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs2, boltDb2) + + runInstance.writeObjectString(t, rootFs2, "one", "content") + listRoot, err := runInstance.list(t, rootFs, "") + require.NoError(t, err) + listRootInner, err := runInstance.list(t, rootFs, innerFolder) + require.NoError(t, err) + listInner, err := rootFs2.List("") + require.NoError(t, err) + + require.Len(t, listRoot, 1) + require.Len(t, listRootInner, 1) + require.Len(t, listInner, 1) +} + +/* TODO: is this testing something? +func TestInternalVfsCache(t *testing.T) { + vfsflags.Opt.DirCacheTime = time.Second * 30 + testSize := int64(524288000) + + vfsflags.Opt.CacheMode = vfs.CacheModeWrites + id := "tiuufo" + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"writes": "true", "info_age": "1h"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + err := rootFs.Mkdir("test") + require.NoError(t, err) + runInstance.writeObjectString(t, rootFs, "test/second", "content") + _, err = rootFs.List("test") + require.NoError(t, err) + + testReader := runInstance.randomReader(t, testSize) + writeCh := make(chan interface{}) + //write2Ch := make(chan interface{}) + readCh := make(chan interface{}) + cacheCh := make(chan interface{}) + // write the main file + go func() { + defer func() { + writeCh <- true + }() + + log.Printf("========== started writing file 'test/one'") + runInstance.writeRemoteReader(t, rootFs, "test/one", testReader) + log.Printf("========== done writing file 'test/one'") + }() + // routine to check which cache has what, autostarts + go func() { + for { + select { + case <-cacheCh: + log.Printf("========== finished checking caches") + return + default: + } + li2 := [2]string{path.Join("test", "one"), path.Join("test", "second")} + for _, r := range li2 { + var err error + ci, err := ioutil.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r)))) + if err != nil || len(ci) == 0 { + log.Printf("========== '%v' not in cache", r) + } else { + log.Printf("========== '%v' IN CACHE", r) + } + _, err = os.Stat(path.Join(runInstance.vfsCachePath, id, r)) + if err != nil { + log.Printf("========== '%v' not in vfs", r) + } else { + log.Printf("========== '%v' IN VFS", r) + } + } + time.Sleep(time.Second * 10) + } + }() + // routine to list, autostarts + go func() { + for { + select { + case <-readCh: + log.Printf("========== finished checking listings and readings") + return + default: + } + li, err := runInstance.list(t, rootFs, "test") + if err != nil { + log.Printf("========== error listing 'test' folder: %v", err) + } else { + log.Printf("========== list 'test' folder count: %v", len(li)) + } + + time.Sleep(time.Second * 10) + } + }() + + // wait for main file to be written + <-writeCh + log.Printf("========== waiting for VFS to expire") + time.Sleep(time.Second * 120) + + // try a final read + li2 := [2]string{"test/one", "test/second"} + for _, r := range li2 { + _, err := runInstance.readDataFromRemote(t, rootFs, r, int64(0), int64(2), false) + if err != nil { + log.Printf("========== error reading '%v': %v", r, err) + } else { + log.Printf("========== read '%v'", r) + } + } + // close the cache and list checkers + cacheCh <- true + readCh <- true +} +*/ + +func TestInternalObjWrapFsFound(t *testing.T) { + id := fmt.Sprintf("tiowff%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + wrappedFs := cfs.UnWrap() + + var testData []byte + if runInstance.rootIsCrypt { + testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64) + require.NoError(t, err) + } else { + testData = []byte("test content") + } + + runInstance.writeObjectBytes(t, wrappedFs, runInstance.encryptRemoteIfNeeded(t, "test"), testData) + listRoot, err := runInstance.list(t, rootFs, "") + require.NoError(t, err) + require.Len(t, listRoot, 1) + + cachedData, err := runInstance.readDataFromRemote(t, rootFs, "test", 0, int64(len([]byte("test content"))), false) + require.NoError(t, err) + require.Equal(t, "test content", string(cachedData)) + + err = runInstance.rm(t, rootFs, "test") + require.NoError(t, err) + listRoot, err = runInstance.list(t, rootFs, "") + require.NoError(t, err) + require.Len(t, listRoot, 0) +} + +func TestInternalObjNotFound(t *testing.T) { + id := fmt.Sprintf("tionf%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + obj, err := rootFs.NewObject("404") + require.Error(t, err) + require.Nil(t, obj) +} + +func TestInternalRemoteWrittenFileFoundInMount(t *testing.T) { + if !runInstance.useMount { + t.Skip("test needs mount mode") + } + id := fmt.Sprintf("tirwffim%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + + var testData []byte + if runInstance.rootIsCrypt { + testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64) + require.NoError(t, err) + } else { + testData = []byte("test content") + } + + runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test"), testData) + data, err := runInstance.readDataFromRemote(t, rootFs, "test", 0, int64(len([]byte("test content"))), false) + require.NoError(t, err) + require.Equal(t, "test content", string(data)) +} + +func TestInternalCachedWrittenContentMatches(t *testing.T) { + id := fmt.Sprintf("ticwcm%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + chunkSize := cfs.ChunkSize() + + // create some rand test data + testData := randStringBytes(int(chunkSize*4 + chunkSize/2)) + + // write the object + runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) + + // check sample of data from in-file + sampleStart := chunkSize / 2 + sampleEnd := chunkSize + testSample := testData[sampleStart:sampleEnd] + checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", sampleStart, sampleEnd, false) + require.NoError(t, err) + require.Equal(t, int64(len(checkSample)), sampleEnd-sampleStart) + require.Equal(t, checkSample, testSample) +} + +func TestInternalDoubleWrittenContentMatches(t *testing.T) { + id := fmt.Sprintf("tidwcm%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + // write the object + runInstance.writeRemoteString(t, rootFs, "one", "one content") + err := runInstance.updateData(t, rootFs, "one", "one content", " updated") + require.NoError(t, err) + err = runInstance.updateData(t, rootFs, "one", "one content updated", " double") + require.NoError(t, err) + + // check sample of data from in-file + data, err := runInstance.readDataFromRemote(t, rootFs, "one", int64(0), int64(len("one content updated double")), true) + require.NoError(t, err) + require.Equal(t, "one content updated double", string(data)) +} + +func TestInternalCachedUpdatedContentMatches(t *testing.T) { + id := fmt.Sprintf("ticucm%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + var err error + + // create some rand test data + var testData1 []byte + var testData2 []byte + if runInstance.rootIsCrypt { + testData1, err = base64.StdEncoding.DecodeString(cryptedTextBase64) + require.NoError(t, err) + testData2, err = base64.StdEncoding.DecodeString(cryptedText2Base64) + require.NoError(t, err) + } else { + testData1 = []byte(fstest.RandomString(100)) + testData2 = []byte(fstest.RandomString(200)) + } + + // write the object + o := runInstance.updateObjectRemote(t, rootFs, "data.bin", testData1, testData2) + require.Equal(t, o.Size(), int64(len(testData2))) + + // check data from in-file + checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(len(testData2)), false) + require.NoError(t, err) + require.Equal(t, checkSample, testData2) +} + +func TestInternalWrappedWrittenContentMatches(t *testing.T) { + id := fmt.Sprintf("tiwwcm%v", time.Now().Unix()) + vfsflags.Opt.DirCacheTime = time.Second + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + if runInstance.rootIsCrypt { + t.Skip("test skipped with crypt remote") + } + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + chunkSize := cfs.ChunkSize() + + // create some rand test data + testSize := chunkSize*4 + chunkSize/2 + testData := randStringBytes(int(testSize)) + + // write the object + o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData) + require.Equal(t, o.Size(), int64(testSize)) + time.Sleep(time.Second * 3) + + checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false) + require.NoError(t, err) + require.Equal(t, int64(len(checkSample)), o.Size()) + + for i := 0; i < len(checkSample); i++ { + require.Equal(t, testData[i], checkSample[i]) + } +} + +func TestInternalLargeWrittenContentMatches(t *testing.T) { + id := fmt.Sprintf("tilwcm%v", time.Now().Unix()) + vfsflags.Opt.DirCacheTime = time.Second + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + if runInstance.rootIsCrypt { + t.Skip("test skipped with crypt remote") + } + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + chunkSize := cfs.ChunkSize() + + // create some rand test data + testSize := chunkSize*10 + chunkSize/2 + testData := randStringBytes(int(testSize)) + + // write the object + runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData) + time.Sleep(time.Second * 3) + + readData, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false) + require.NoError(t, err) + for i := 0; i < len(readData); i++ { + require.Equalf(t, testData[i], readData[i], "at byte %v", i) + } +} + +func TestInternalWrappedFsChangeNotSeen(t *testing.T) { + id := fmt.Sprintf("tiwfcns%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + chunkSize := cfs.ChunkSize() + + // create some rand test data + testData := randStringBytes(int(chunkSize*4 + chunkSize/2)) + runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) + + // update in the wrapped fs + originalSize, err := runInstance.size(t, rootFs, "data.bin") + require.NoError(t, err) + log.Printf("original size: %v", originalSize) + + o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin")) + require.NoError(t, err) + expectedSize := int64(len([]byte("test content"))) + var data2 []byte + if runInstance.rootIsCrypt { + data2, err = base64.StdEncoding.DecodeString(cryptedText3Base64) + require.NoError(t, err) + expectedSize = expectedSize + 1 // FIXME newline gets in, likely test data issue + } else { + data2 = []byte("test content") + } + objInfo := object.NewStaticObjectInfo(runInstance.encryptRemoteIfNeeded(t, "data.bin"), time.Now(), int64(len(data2)), true, nil, cfs.UnWrap()) + err = o.Update(bytes.NewReader(data2), objInfo) + require.NoError(t, err) + require.Equal(t, int64(len(data2)), o.Size()) + log.Printf("updated size: %v", len(data2)) + + // get a new instance from the cache + if runInstance.wrappedIsExternal { + err = runInstance.retryBlock(func() error { + coSize, err := runInstance.size(t, rootFs, "data.bin") + if err != nil { + return err + } + if coSize != expectedSize { + return errors.Errorf("%v <> %v", coSize, expectedSize) + } + return nil + }, 12, time.Second*10) + require.NoError(t, err) + } else { + coSize, err := runInstance.size(t, rootFs, "data.bin") + require.NoError(t, err) + require.NotEqual(t, coSize, expectedSize) + } +} + +func TestInternalMoveWithNotify(t *testing.T) { + id := fmt.Sprintf("timwn%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + if !runInstance.wrappedIsExternal { + t.Skipf("Not external") + } + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + + srcName := runInstance.encryptRemoteIfNeeded(t, "test") + "/" + runInstance.encryptRemoteIfNeeded(t, "one") + "/" + runInstance.encryptRemoteIfNeeded(t, "data.bin") + dstName := runInstance.encryptRemoteIfNeeded(t, "test") + "/" + runInstance.encryptRemoteIfNeeded(t, "second") + "/" + runInstance.encryptRemoteIfNeeded(t, "data.bin") + // create some rand test data + var testData []byte + if runInstance.rootIsCrypt { + testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64) + require.NoError(t, err) + } else { + testData = []byte("test content") + } + _ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test")) + _ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/one")) + _ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/second")) + srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData) + + // list in mount + _, err = runInstance.list(t, rootFs, "test") + require.NoError(t, err) + _, err = runInstance.list(t, rootFs, "test/one") + require.NoError(t, err) + + // move file + _, err = cfs.UnWrap().Features().Move(srcObj, dstName) + require.NoError(t, err) + + err = runInstance.retryBlock(func() error { + li, err := runInstance.list(t, rootFs, "test") + if err != nil { + log.Printf("err: %v", err) + return err + } + if len(li) != 2 { + log.Printf("not expected listing /test: %v", li) + return errors.Errorf("not expected listing /test: %v", li) + } + + li, err = runInstance.list(t, rootFs, "test/one") + if err != nil { + log.Printf("err: %v", err) + return err + } + if len(li) != 0 { + log.Printf("not expected listing /test/one: %v", li) + return errors.Errorf("not expected listing /test/one: %v", li) + } + + li, err = runInstance.list(t, rootFs, "test/second") + if err != nil { + log.Printf("err: %v", err) + return err + } + if len(li) != 1 { + log.Printf("not expected listing /test/second: %v", li) + return errors.Errorf("not expected listing /test/second: %v", li) + } + if fi, ok := li[0].(os.FileInfo); ok { + if fi.Name() != "data.bin" { + log.Printf("not expected name: %v", fi.Name()) + return errors.Errorf("not expected name: %v", fi.Name()) + } + } else if di, ok := li[0].(fs.DirEntry); ok { + if di.Remote() != "test/second/data.bin" { + log.Printf("not expected remote: %v", di.Remote()) + return errors.Errorf("not expected remote: %v", di.Remote()) + } + } else { + log.Printf("unexpected listing: %v", li) + return errors.Errorf("unexpected listing: %v", li) + } + + log.Printf("complete listing: %v", li) + return nil + }, 12, time.Second*10) + require.NoError(t, err) +} + +func TestInternalNotifyCreatesEmptyParts(t *testing.T) { + id := fmt.Sprintf("tincep%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + if !runInstance.wrappedIsExternal { + t.Skipf("Not external") + } + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + + srcName := runInstance.encryptRemoteIfNeeded(t, "test") + "/" + runInstance.encryptRemoteIfNeeded(t, "one") + "/" + runInstance.encryptRemoteIfNeeded(t, "test") + dstName := runInstance.encryptRemoteIfNeeded(t, "test") + "/" + runInstance.encryptRemoteIfNeeded(t, "one") + "/" + runInstance.encryptRemoteIfNeeded(t, "test2") + // create some rand test data + var testData []byte + if runInstance.rootIsCrypt { + testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64) + require.NoError(t, err) + } else { + testData = []byte("test content") + } + err = rootFs.Mkdir("test") + require.NoError(t, err) + err = rootFs.Mkdir("test/one") + require.NoError(t, err) + srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData) + + // list in mount + _, err = runInstance.list(t, rootFs, "test") + require.NoError(t, err) + _, err = runInstance.list(t, rootFs, "test/one") + require.NoError(t, err) + + found := boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"))) + require.True(t, found) + boltDb.Purge() + found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"))) + require.False(t, found) + + // move file + _, err = cfs.UnWrap().Features().Move(srcObj, dstName) + require.NoError(t, err) + + err = runInstance.retryBlock(func() error { + found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"))) + if !found { + log.Printf("not found /test") + return errors.Errorf("not found /test") + } + found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one"))) + if !found { + log.Printf("not found /test/one") + return errors.Errorf("not found /test/one") + } + found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one"), runInstance.encryptRemoteIfNeeded(t, "test2"))) + if !found { + log.Printf("not found /test/one/test2") + return errors.Errorf("not found /test/one/test2") + } + li, err := runInstance.list(t, rootFs, "test/one") + if err != nil { + log.Printf("err: %v", err) + return err + } + if len(li) != 1 { + log.Printf("not expected listing /test/one: %v", li) + return errors.Errorf("not expected listing /test/one: %v", li) + } + if fi, ok := li[0].(os.FileInfo); ok { + if fi.Name() != "test2" { + log.Printf("not expected name: %v", fi.Name()) + return errors.Errorf("not expected name: %v", fi.Name()) + } + } else if di, ok := li[0].(fs.DirEntry); ok { + if di.Remote() != "test/one/test2" { + log.Printf("not expected remote: %v", di.Remote()) + return errors.Errorf("not expected remote: %v", di.Remote()) + } + } else { + log.Printf("unexpected listing: %v", li) + return errors.Errorf("unexpected listing: %v", li) + } + log.Printf("complete listing /test/one/test2") + return nil + }, 12, time.Second*10) + require.NoError(t, err) +} + +func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) { + id := fmt.Sprintf("ticsadcf%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + chunkSize := cfs.ChunkSize() + + // create some rand test data + testData := randStringBytes(int(chunkSize*4 + chunkSize/2)) + runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) + + // update in the wrapped fs + o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin")) + require.NoError(t, err) + wrappedTime := time.Now().Add(-1 * time.Hour) + err = o.SetModTime(wrappedTime) + require.NoError(t, err) + + // get a new instance from the cache + co, err := rootFs.NewObject("data.bin") + require.NoError(t, err) + require.NotEqual(t, o.ModTime().String(), co.ModTime().String()) + + cfs.DirCacheFlush() // flush the cache + + // get a new instance from the cache + co, err = rootFs.NewObject("data.bin") + require.NoError(t, err) + require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix()) +} + +func TestInternalChangeSeenAfterRc(t *testing.T) { + rcflags.Opt.Enabled = true + rc.Start(&rcflags.Opt) + + id := fmt.Sprintf("ticsarc%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + if !runInstance.useMount { + t.Skipf("needs mount") + } + if !runInstance.wrappedIsExternal { + t.Skipf("needs drive") + } + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + chunkSize := cfs.ChunkSize() + + // create some rand test data + testData := randStringBytes(int(chunkSize*4 + chunkSize/2)) + runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) + + // update in the wrapped fs + o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin")) + require.NoError(t, err) + wrappedTime := time.Now().Add(-1 * time.Hour) + err = o.SetModTime(wrappedTime) + require.NoError(t, err) + + // get a new instance from the cache + co, err := rootFs.NewObject("data.bin") + require.NoError(t, err) + require.NotEqual(t, o.ModTime().String(), co.ModTime().String()) + + m := make(map[string]string) + res, err := http.Post(fmt.Sprintf("http://localhost:5572/cache/expire?remote=%s", "data.bin"), "application/json; charset=utf-8", strings.NewReader("")) + require.NoError(t, err) + defer func() { + _ = res.Body.Close() + }() + _ = json.NewDecoder(res.Body).Decode(&m) + require.Contains(t, m, "status") + require.Contains(t, m, "message") + require.Equal(t, "ok", m["status"]) + require.Contains(t, m["message"], "cached file cleared") + + // get a new instance from the cache + co, err = rootFs.NewObject("data.bin") + require.NoError(t, err) + require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix()) + li1, err := runInstance.list(t, rootFs, "") + + // create some rand test data + testData2 := randStringBytes(int(chunkSize)) + runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test2"), testData2) + + // list should have 1 item only + li1, err = runInstance.list(t, rootFs, "") + require.Len(t, li1, 1) + + m = make(map[string]string) + res2, err := http.Post("http://localhost:5572/cache/expire?remote=/", "application/json; charset=utf-8", strings.NewReader("")) + require.NoError(t, err) + defer func() { + _ = res2.Body.Close() + }() + _ = json.NewDecoder(res2.Body).Decode(&m) + require.Contains(t, m, "status") + require.Contains(t, m, "message") + require.Equal(t, "ok", m["status"]) + require.Contains(t, m["message"], "cached directory cleared") + + // list should have 2 items now + li2, err := runInstance.list(t, rootFs, "") + require.Len(t, li2, 2) +} + +func TestInternalCacheWrites(t *testing.T) { + id := "ticw" + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"writes": "true"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + chunkSize := cfs.ChunkSize() + + // create some rand test data + earliestTime := time.Now() + testData := randStringBytes(int(chunkSize*4 + chunkSize/2)) + runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) + expectedTs := time.Now() + ts, err := boltDb.GetChunkTs(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "data.bin")), 0) + require.NoError(t, err) + require.WithinDuration(t, expectedTs, ts, expectedTs.Sub(earliestTime)) +} + +func TestInternalMaxChunkSizeRespected(t *testing.T) { + id := fmt.Sprintf("timcsr%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"workers": "1"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + chunkSize := cfs.ChunkSize() + totalChunks := 20 + + // create some rand test data + testData := randStringBytes(int(int64(totalChunks-1)*chunkSize + chunkSize/2)) + runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) + o, err := cfs.NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin")) + require.NoError(t, err) + co, ok := o.(*cache.Object) + require.True(t, ok) + + for i := 0; i < 4; i++ { // read first 4 + _ = runInstance.readDataFromObj(t, co, chunkSize*int64(i), chunkSize*int64(i+1), false) + } + cfs.CleanUpCache(true) + // the last 2 **must** be in the cache + require.True(t, boltDb.HasChunk(co, chunkSize*2)) + require.True(t, boltDb.HasChunk(co, chunkSize*3)) + + for i := 4; i < 6; i++ { // read next 2 + _ = runInstance.readDataFromObj(t, co, chunkSize*int64(i), chunkSize*int64(i+1), false) + } + cfs.CleanUpCache(true) + // the last 2 **must** be in the cache + require.True(t, boltDb.HasChunk(co, chunkSize*4)) + require.True(t, boltDb.HasChunk(co, chunkSize*5)) +} + +func TestInternalExpiredEntriesRemoved(t *testing.T) { + id := fmt.Sprintf("tieer%v", time.Now().Unix()) + vfsflags.Opt.DirCacheTime = time.Second * 4 // needs to be lower than the defined + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, map[string]string{"info_age": "5s"}, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + + // create some rand test data + runInstance.writeRemoteString(t, rootFs, "one", "one content") + runInstance.mkdir(t, rootFs, "test") + runInstance.writeRemoteString(t, rootFs, "test/second", "second content") + + l, err := runInstance.list(t, rootFs, "test") + require.NoError(t, err) + require.Len(t, l, 1) + + err = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/third")) + require.NoError(t, err) + + l, err = runInstance.list(t, rootFs, "test") + require.NoError(t, err) + require.Len(t, l, 1) + + err = runInstance.retryBlock(func() error { + l, err = runInstance.list(t, rootFs, "test") + if err != nil { + return err + } + if len(l) != 2 { + return errors.New("list is not 2") + } + return nil + }, 10, time.Second) + require.NoError(t, err) +} + +func TestInternalBug2117(t *testing.T) { + vfsflags.Opt.DirCacheTime = time.Second * 10 + + id := fmt.Sprintf("tib2117%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, + map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + if runInstance.rootIsCrypt { + t.Skipf("skipping crypt") + } + + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + + err = cfs.UnWrap().Mkdir("test") + require.NoError(t, err) + for i := 1; i <= 4; i++ { + err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d", i)) + require.NoError(t, err) + + for j := 1; j <= 4; j++ { + err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d/dir%d", i, j)) + require.NoError(t, err) + + runInstance.writeObjectString(t, cfs.UnWrap(), fmt.Sprintf("test/dir%d/dir%d/test.txt", i, j), "test") + } + } + + di, err := runInstance.list(t, rootFs, "test/dir1/dir2") + require.NoError(t, err) + log.Printf("len: %v", len(di)) + require.Len(t, di, 1) + + time.Sleep(time.Second * 30) + + di, err = runInstance.list(t, rootFs, "test/dir1/dir2") + require.NoError(t, err) + log.Printf("len: %v", len(di)) + require.Len(t, di, 1) + + di, err = runInstance.list(t, rootFs, "test/dir1") + require.NoError(t, err) + log.Printf("len: %v", len(di)) + require.Len(t, di, 4) + + di, err = runInstance.list(t, rootFs, "test") + require.NoError(t, err) + log.Printf("len: %v", len(di)) + require.Len(t, di, 4) +} + +// run holds the remotes for a test run +type run struct { + okDiff time.Duration + runDefaultCfgMap configmap.Simple + mntDir string + tmpUploadDir string + useMount bool + isMounted bool + rootIsCrypt bool + wrappedIsExternal bool + unmountFn func() error + unmountRes chan error + vfs *vfs.VFS + tempFiles []*os.File + dbPath string + chunkPath string + vfsCachePath string +} + +func newRun() *run { + var err error + r := &run{ + okDiff: time.Second * 9, // really big diff here but the build machines seem to be slow. need a different way for this + useMount: useMount, + isMounted: false, + } + + // Read in all the defaults for all the options + fsInfo, err := fs.Find("cache") + if err != nil { + panic(fmt.Sprintf("Couldn't find cache remote: %v", err)) + } + r.runDefaultCfgMap = configmap.Simple{} + for _, option := range fsInfo.Options { + r.runDefaultCfgMap.Set(option.Name, fmt.Sprint(option.Default)) + } + + if mountDir == "" { + if runtime.GOOS != "windows" { + r.mntDir, err = ioutil.TempDir("", "rclonecache-mount") + if err != nil { + log.Fatalf("Failed to create mount dir: %v", err) + return nil + } + } else { + // Find a free drive letter + drive := "" + for letter := 'E'; letter <= 'Z'; letter++ { + drive = string(letter) + ":" + _, err := os.Stat(drive + "\\") + if os.IsNotExist(err) { + goto found + } + } + log.Print("Couldn't find free drive letter for test") + found: + r.mntDir = drive + } + } else { + r.mntDir = mountDir + } + log.Printf("Mount Dir: %v", r.mntDir) + + if uploadDir == "" { + r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp") + if err != nil { + log.Fatalf("Failed to create temp dir: %v", err) + } + } else { + r.tmpUploadDir = uploadDir + } + log.Printf("Temp Upload Dir: %v", r.tmpUploadDir) + + return r +} + +func (r *run) encryptRemoteIfNeeded(t *testing.T, remote string) string { + if !runInstance.rootIsCrypt || len(decryptedToEncryptedRemotes) == 0 { + return remote + } + + enc, ok := decryptedToEncryptedRemotes[remote] + if !ok { + t.Fatalf("Failed to find decrypted -> encrypted mapping for '%v'", remote) + return remote + } + return enc +} + +func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, cfg map[string]string, flags map[string]string) (fs.Fs, *cache.Persistent) { + fstest.Initialise() + remoteExists := false + for _, s := range config.FileSections() { + if s == remote { + remoteExists = true + } + } + if !remoteExists && needRemote { + t.Skipf("Need remote (%v) to exist", remote) + return nil, nil + } + + // if the remote doesn't exist, create a new one with a local one for it + // identify which is the cache remote (it can be wrapped by a crypt too) + rootIsCrypt := false + cacheRemote := remote + if !remoteExists { + localRemote := remote + "-local" + config.FileSet(localRemote, "type", "local") + config.FileSet(localRemote, "nounc", "true") + config.FileSet(remote, "type", "cache") + config.FileSet(remote, "remote", localRemote+":/var/tmp/"+localRemote) + } else { + remoteType := config.FileGet(remote, "type", "") + if remoteType == "" { + t.Skipf("skipped due to invalid remote type for %v", remote) + return nil, nil + } + if remoteType != "cache" { + if remoteType == "crypt" { + rootIsCrypt = true + config.FileSet(remote, "password", cryptPassword1) + config.FileSet(remote, "password2", cryptPassword2) + } + remoteRemote := config.FileGet(remote, "remote", "") + if remoteRemote == "" { + t.Skipf("skipped due to invalid remote wrapper for %v", remote) + return nil, nil + } + remoteRemoteParts := strings.Split(remoteRemote, ":") + remoteWrapping := remoteRemoteParts[0] + remoteType := config.FileGet(remoteWrapping, "type", "") + if remoteType != "cache" { + t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType) + return nil, nil + } + cacheRemote = remoteWrapping + } + } + runInstance.rootIsCrypt = rootIsCrypt + runInstance.dbPath = filepath.Join(config.CacheDir, "cache-backend", cacheRemote+".db") + runInstance.chunkPath = filepath.Join(config.CacheDir, "cache-backend", cacheRemote) + runInstance.vfsCachePath = filepath.Join(config.CacheDir, "vfs", remote) + boltDb, err := cache.GetPersistent(runInstance.dbPath, runInstance.chunkPath, &cache.Features{PurgeDb: true}) + require.NoError(t, err) + + fs.Config.LowLevelRetries = 1 + + m := configmap.Simple{} + for k, v := range r.runDefaultCfgMap { + m.Set(k, v) + } + for k, v := range flags { + m.Set(k, v) + } + + // Instantiate root + if purge { + boltDb.PurgeTempUploads() + _ = os.RemoveAll(path.Join(runInstance.tmpUploadDir, id)) + } + f, err := cache.NewFs(remote, id, m) + require.NoError(t, err) + cfs, err := r.getCacheFs(f) + require.NoError(t, err) + _, isCache := cfs.Features().UnWrap().(*cache.Fs) + _, isCrypt := cfs.Features().UnWrap().(*crypt.Fs) + _, isLocal := cfs.Features().UnWrap().(*local.Fs) + if isCache || isCrypt || isLocal { + r.wrappedIsExternal = false + } else { + r.wrappedIsExternal = true + } + + if purge { + _ = f.Features().Purge() + require.NoError(t, err) + } + err = f.Mkdir("") + require.NoError(t, err) + if r.useMount && !r.isMounted { + r.mountFs(t, f) + } + + return f, boltDb +} + +func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) { + if r.useMount && r.isMounted { + r.unmountFs(t, f) + } + + err := f.Features().Purge() + require.NoError(t, err) + cfs, err := r.getCacheFs(f) + require.NoError(t, err) + cfs.StopBackgroundRunners() + + if r.useMount && runtime.GOOS != "windows" { + err = os.RemoveAll(r.mntDir) + require.NoError(t, err) + } + err = os.RemoveAll(r.tmpUploadDir) + require.NoError(t, err) + + for _, f := range r.tempFiles { + _ = f.Close() + _ = os.Remove(f.Name()) + } + r.tempFiles = nil + debug.FreeOSMemory() +} + +func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser { + chunk := int64(1024) + cnt := size / chunk + left := size % chunk + f, err := ioutil.TempFile("", "rclonecache-tempfile") + require.NoError(t, err) + + for i := 0; i < int(cnt); i++ { + data := randStringBytes(int(chunk)) + _, _ = f.Write(data) + } + data := randStringBytes(int(left)) + _, _ = f.Write(data) + _, _ = f.Seek(int64(0), io.SeekStart) + r.tempFiles = append(r.tempFiles, f) + + return f +} + +func (r *run) writeRemoteRandomBytes(t *testing.T, f fs.Fs, p string, size int64) string { + remote := path.Join(p, strconv.Itoa(rand.Int())+".bin") + // create some rand test data + testData := randStringBytes(int(size)) + + r.writeRemoteBytes(t, f, remote, testData) + return remote +} + +func (r *run) writeObjectRandomBytes(t *testing.T, f fs.Fs, p string, size int64) fs.Object { + remote := path.Join(p, strconv.Itoa(rand.Int())+".bin") + // create some rand test data + testData := randStringBytes(int(size)) + + return r.writeObjectBytes(t, f, remote, testData) +} + +func (r *run) writeRemoteString(t *testing.T, f fs.Fs, remote, content string) { + r.writeRemoteBytes(t, f, remote, []byte(content)) +} + +func (r *run) writeObjectString(t *testing.T, f fs.Fs, remote, content string) fs.Object { + return r.writeObjectBytes(t, f, remote, []byte(content)) +} + +func (r *run) writeRemoteBytes(t *testing.T, f fs.Fs, remote string, data []byte) { + var err error + + if r.useMount { + err = r.retryBlock(func() error { + return ioutil.WriteFile(path.Join(r.mntDir, remote), data, 0600) + }, 3, time.Second*3) + require.NoError(t, err) + r.vfs.WaitForWriters(10 * time.Second) + } else { + r.writeObjectBytes(t, f, remote, data) + } +} + +func (r *run) writeRemoteReader(t *testing.T, f fs.Fs, remote string, in io.ReadCloser) { + defer func() { + _ = in.Close() + }() + + if r.useMount { + out, err := os.Create(path.Join(r.mntDir, remote)) + require.NoError(t, err) + defer func() { + _ = out.Close() + }() + + _, err = io.Copy(out, in) + require.NoError(t, err) + r.vfs.WaitForWriters(10 * time.Second) + } else { + r.writeObjectReader(t, f, remote, in) + } +} + +func (r *run) writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte) fs.Object { + in := bytes.NewReader(data) + _ = r.writeObjectReader(t, f, remote, in) + o, err := f.NewObject(remote) + require.NoError(t, err) + require.Equal(t, int64(len(data)), o.Size()) + return o +} + +func (r *run) writeObjectReader(t *testing.T, f fs.Fs, remote string, in io.Reader) fs.Object { + modTime := time.Now() + objInfo := object.NewStaticObjectInfo(remote, modTime, -1, true, nil, f) + obj, err := f.Put(in, objInfo) + require.NoError(t, err) + if r.useMount { + r.vfs.WaitForWriters(10 * time.Second) + } + + return obj +} + +func (r *run) updateObjectRemote(t *testing.T, f fs.Fs, remote string, data1 []byte, data2 []byte) fs.Object { + var err error + var obj fs.Object + + if r.useMount { + err = ioutil.WriteFile(path.Join(r.mntDir, remote), data1, 0600) + require.NoError(t, err) + r.vfs.WaitForWriters(10 * time.Second) + err = ioutil.WriteFile(path.Join(r.mntDir, remote), data2, 0600) + require.NoError(t, err) + r.vfs.WaitForWriters(10 * time.Second) + obj, err = f.NewObject(remote) + } else { + in1 := bytes.NewReader(data1) + in2 := bytes.NewReader(data2) + objInfo1 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data1)), true, nil, f) + objInfo2 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data2)), true, nil, f) + + obj, err = f.Put(in1, objInfo1) + require.NoError(t, err) + obj, err = f.NewObject(remote) + require.NoError(t, err) + err = obj.Update(in2, objInfo2) + } + require.NoError(t, err) + + return obj +} + +func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, end int64, noLengthCheck bool) ([]byte, error) { + size := end - offset + checkSample := make([]byte, size) + + if r.useMount { + f, err := os.Open(path.Join(r.mntDir, remote)) + defer func() { + _ = f.Close() + }() + if err != nil { + return checkSample, err + } + _, _ = f.Seek(offset, io.SeekStart) + totalRead, err := io.ReadFull(f, checkSample) + checkSample = checkSample[:totalRead] + if err == io.EOF || err == io.ErrUnexpectedEOF { + err = nil + } + if err != nil { + return checkSample, err + } + } else { + co, err := f.NewObject(remote) + if err != nil { + return checkSample, err + } + checkSample = r.readDataFromObj(t, co, offset, end, noLengthCheck) + } + if !noLengthCheck && size != int64(len(checkSample)) { + return checkSample, errors.Errorf("read size doesn't match expected: %v <> %v", len(checkSample), size) + } + return checkSample, nil +} + +func (r *run) readDataFromObj(t *testing.T, o fs.Object, offset, end int64, noLengthCheck bool) []byte { + size := end - offset + checkSample := make([]byte, size) + reader, err := o.Open(&fs.SeekOption{Offset: offset}) + require.NoError(t, err) + totalRead, err := io.ReadFull(reader, checkSample) + if (err == io.EOF || err == io.ErrUnexpectedEOF) && noLengthCheck { + err = nil + checkSample = checkSample[:totalRead] + } + require.NoError(t, err, "with string -%v-", string(checkSample)) + _ = reader.Close() + return checkSample +} + +func (r *run) mkdir(t *testing.T, f fs.Fs, remote string) { + var err error + if r.useMount { + err = os.Mkdir(path.Join(r.mntDir, remote), 0700) + } else { + err = f.Mkdir(remote) + } + require.NoError(t, err) +} + +func (r *run) rm(t *testing.T, f fs.Fs, remote string) error { + var err error + + if r.useMount { + err = os.Remove(path.Join(r.mntDir, remote)) + } else { + var obj fs.Object + obj, err = f.NewObject(remote) + if err != nil { + err = f.Rmdir(remote) + } else { + err = obj.Remove() + } + } + + return err +} + +func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error) { + var err error + var l []interface{} + if r.useMount { + var list []os.FileInfo + list, err = ioutil.ReadDir(path.Join(r.mntDir, remote)) + for _, ll := range list { + l = append(l, ll) + } + } else { + var list fs.DirEntries + list, err = f.List(remote) + for _, ll := range list { + l = append(l, ll) + } + } + return l, err +} + +func (r *run) listPath(t *testing.T, f fs.Fs, remote string) []string { + var err error + var l []string + if r.useMount { + var list []os.FileInfo + list, err = ioutil.ReadDir(path.Join(r.mntDir, remote)) + for _, ll := range list { + l = append(l, ll.Name()) + } + } else { + var list fs.DirEntries + list, err = f.List(remote) + for _, ll := range list { + l = append(l, ll.Remote()) + } + } + require.NoError(t, err) + return l +} + +func (r *run) copyFile(t *testing.T, f fs.Fs, src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { + _ = in.Close() + }() + + out, err := os.Create(dst) + if err != nil { + return err + } + defer func() { + _ = out.Close() + }() + + _, err = io.Copy(out, in) + return err +} + +func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error { + var err error + + if runInstance.useMount { + err = os.Rename(path.Join(runInstance.mntDir, src), path.Join(runInstance.mntDir, dst)) + if err != nil { + return err + } + r.vfs.WaitForWriters(10 * time.Second) + } else if rootFs.Features().DirMove != nil { + err = rootFs.Features().DirMove(rootFs, src, dst) + if err != nil { + return err + } + } else { + t.Logf("DirMove not supported by %v", rootFs) + return errNotSupported + } + + return err +} + +func (r *run) move(t *testing.T, rootFs fs.Fs, src, dst string) error { + var err error + + if runInstance.useMount { + err = os.Rename(path.Join(runInstance.mntDir, src), path.Join(runInstance.mntDir, dst)) + if err != nil { + return err + } + r.vfs.WaitForWriters(10 * time.Second) + } else if rootFs.Features().Move != nil { + obj1, err := rootFs.NewObject(src) + if err != nil { + return err + } + _, err = rootFs.Features().Move(obj1, dst) + if err != nil { + return err + } + } else { + t.Logf("Move not supported by %v", rootFs) + return errNotSupported + } + + return err +} + +func (r *run) copy(t *testing.T, rootFs fs.Fs, src, dst string) error { + var err error + + if r.useMount { + err = r.copyFile(t, rootFs, path.Join(r.mntDir, src), path.Join(r.mntDir, dst)) + if err != nil { + return err + } + r.vfs.WaitForWriters(10 * time.Second) + } else if rootFs.Features().Copy != nil { + obj, err := rootFs.NewObject(src) + if err != nil { + return err + } + _, err = rootFs.Features().Copy(obj, dst) + if err != nil { + return err + } + } else { + t.Logf("Copy not supported by %v", rootFs) + return errNotSupported + } + + return err +} + +func (r *run) modTime(t *testing.T, rootFs fs.Fs, src string) (time.Time, error) { + var err error + + if r.useMount { + fi, err := os.Stat(path.Join(runInstance.mntDir, src)) + if err != nil { + return time.Time{}, err + } + return fi.ModTime(), nil + } + obj1, err := rootFs.NewObject(src) + if err != nil { + return time.Time{}, err + } + return obj1.ModTime(), nil +} + +func (r *run) size(t *testing.T, rootFs fs.Fs, src string) (int64, error) { + var err error + + if r.useMount { + fi, err := os.Stat(path.Join(runInstance.mntDir, src)) + if err != nil { + return int64(0), err + } + return fi.Size(), nil + } + obj1, err := rootFs.NewObject(src) + if err != nil { + return int64(0), err + } + return obj1.Size(), nil +} + +func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) error { + var err error + + if r.useMount { + f, err := os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer func() { + _ = f.Close() + r.vfs.WaitForWriters(10 * time.Second) + }() + _, err = f.WriteString(data + append) + } else { + obj1, err := rootFs.NewObject(src) + if err != nil { + return err + } + data1 := []byte(data + append) + r := bytes.NewReader(data1) + objInfo1 := object.NewStaticObjectInfo(src, time.Now(), int64(len(data1)), true, nil, rootFs) + err = obj1.Update(r, objInfo1) + } + + return err +} + +func (r *run) cleanSize(t *testing.T, size int64) int64 { + if r.rootIsCrypt { + denominator := int64(65536 + 16) + size = size - 32 + quotient := size / denominator + remainder := size % denominator + return (quotient*65536 + remainder - 16) + } + + return size +} + +func (r *run) listenForBackgroundUpload(t *testing.T, f fs.Fs, remote string) chan error { + cfs, err := r.getCacheFs(f) + require.NoError(t, err) + buCh := cfs.GetBackgroundUploadChannel() + require.NotNil(t, buCh) + maxDuration := time.Minute * 3 + if r.wrappedIsExternal { + maxDuration = time.Minute * 10 + } + + waitCh := make(chan error) + go func() { + var err error + var state cache.BackgroundUploadState + + for i := 0; i < 2; i++ { + select { + case state = <-buCh: + // continue + case <-time.After(maxDuration): + waitCh <- errors.Errorf("Timed out waiting for background upload: %v", remote) + return + } + checkRemote := state.Remote + if r.rootIsCrypt { + cryptFs := f.(*crypt.Fs) + checkRemote, err = cryptFs.DecryptFileName(checkRemote) + if err != nil { + waitCh <- err + return + } + } + if checkRemote == remote && cache.BackgroundUploadStarted != state.Status { + waitCh <- state.Error + return + } + } + waitCh <- errors.Errorf("Too many attempts to wait for the background upload: %v", remote) + }() + return waitCh +} + +func (r *run) completeBackgroundUpload(t *testing.T, remote string, waitCh chan error) { + var err error + maxDuration := time.Minute * 3 + if r.wrappedIsExternal { + maxDuration = time.Minute * 10 + } + select { + case err = <-waitCh: + // continue + case <-time.After(maxDuration): + t.Fatalf("Timed out waiting to complete the background upload %v", remote) + return + } + require.NoError(t, err) +} + +func (r *run) completeAllBackgroundUploads(t *testing.T, f fs.Fs, lastRemote string) { + var state cache.BackgroundUploadState + var err error + + maxDuration := time.Minute * 5 + if r.wrappedIsExternal { + maxDuration = time.Minute * 15 + } + cfs, err := r.getCacheFs(f) + require.NoError(t, err) + buCh := cfs.GetBackgroundUploadChannel() + require.NotNil(t, buCh) + + for { + select { + case state = <-buCh: + checkRemote := state.Remote + if r.rootIsCrypt { + cryptFs := f.(*crypt.Fs) + checkRemote, err = cryptFs.DecryptFileName(checkRemote) + require.NoError(t, err) + } + if checkRemote == lastRemote && cache.BackgroundUploadCompleted == state.Status { + require.NoError(t, state.Error) + return + } + case <-time.After(maxDuration): + t.Fatalf("Timed out waiting to complete the background upload %v", lastRemote) + return + } + } +} + +func (r *run) retryBlock(block func() error, maxRetries int, rate time.Duration) error { + var err error + for i := 0; i < maxRetries; i++ { + err = block() + if err == nil { + return nil + } + time.Sleep(rate) + } + return err +} + +func (r *run) getCacheFs(f fs.Fs) (*cache.Fs, error) { + cfs, ok := f.(*cache.Fs) + if ok { + return cfs, nil + } else { + if f.Features().UnWrap != nil { + cfs, ok := f.Features().UnWrap().(*cache.Fs) + if ok { + return cfs, nil + } + } + } + + return nil, errors.New("didn't found a cache fs") +} + +func randStringBytes(n int) []byte { + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] + } + return b +} + +var ( + _ fs.Fs = (*cache.Fs)(nil) + _ fs.Fs = (*local.Fs)(nil) +) diff --git a/.rclone_repo/backend/cache/cache_mount_unix_test.go b/.rclone_repo/backend/cache/cache_mount_unix_test.go new file mode 100755 index 0000000..2fe7d53 --- /dev/null +++ b/.rclone_repo/backend/cache/cache_mount_unix_test.go @@ -0,0 +1,78 @@ +// +build !plan9,!windows + +package cache_test + +import ( + "os" + "testing" + "time" + + "bazil.org/fuse" + fusefs "bazil.org/fuse/fs" + "github.com/ncw/rclone/cmd/mount" + "github.com/ncw/rclone/cmd/mountlib" + "github.com/ncw/rclone/fs" + "github.com/stretchr/testify/require" +) + +func (r *run) mountFs(t *testing.T, f fs.Fs) { + device := f.Name() + ":" + f.Root() + var options = []fuse.MountOption{ + fuse.MaxReadahead(uint32(mountlib.MaxReadAhead)), + fuse.Subtype("rclone"), + fuse.FSName(device), fuse.VolumeName(device), + fuse.NoAppleDouble(), + fuse.NoAppleXattr(), + //fuse.AllowOther(), + } + err := os.MkdirAll(r.mntDir, os.ModePerm) + require.NoError(t, err) + c, err := fuse.Mount(r.mntDir, options...) + require.NoError(t, err) + filesys := mount.NewFS(f) + server := fusefs.New(c, nil) + + // Serve the mount point in the background returning error to errChan + r.unmountRes = make(chan error, 1) + go func() { + err := server.Serve(filesys) + closeErr := c.Close() + if err == nil { + err = closeErr + } + r.unmountRes <- err + }() + + // check if the mount process has an error to report + <-c.Ready + require.NoError(t, c.MountError) + + r.unmountFn = func() error { + // Shutdown the VFS + filesys.VFS.Shutdown() + return fuse.Unmount(r.mntDir) + } + + r.vfs = filesys.VFS + r.isMounted = true +} + +func (r *run) unmountFs(t *testing.T, f fs.Fs) { + var err error + + for i := 0; i < 4; i++ { + err = r.unmountFn() + if err != nil { + //log.Printf("signal to umount failed - retrying: %v", err) + time.Sleep(3 * time.Second) + continue + } + break + } + require.NoError(t, err) + err = <-r.unmountRes + require.NoError(t, err) + err = r.vfs.CleanUp() + require.NoError(t, err) + r.isMounted = false +} diff --git a/.rclone_repo/backend/cache/cache_mount_windows_test.go b/.rclone_repo/backend/cache/cache_mount_windows_test.go new file mode 100755 index 0000000..44c619a --- /dev/null +++ b/.rclone_repo/backend/cache/cache_mount_windows_test.go @@ -0,0 +1,124 @@ +// +build windows + +package cache_test + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/billziss-gh/cgofuse/fuse" + "github.com/ncw/rclone/cmd/cmount" + "github.com/ncw/rclone/cmd/mountlib" + "github.com/ncw/rclone/fs" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +// waitFor runs fn() until it returns true or the timeout expires +func waitFor(fn func() bool) (ok bool) { + const totalWait = 10 * time.Second + const individualWait = 10 * time.Millisecond + for i := 0; i < int(totalWait/individualWait); i++ { + ok = fn() + if ok { + return ok + } + time.Sleep(individualWait) + } + return false +} + +func (r *run) mountFs(t *testing.T, f fs.Fs) { + // FIXME implement cmount + t.Skip("windows not supported yet") + + device := f.Name() + ":" + f.Root() + options := []string{ + "-o", "fsname=" + device, + "-o", "subtype=rclone", + "-o", fmt.Sprintf("max_readahead=%d", mountlib.MaxReadAhead), + "-o", "uid=-1", + "-o", "gid=-1", + "-o", "allow_other", + // This causes FUSE to supply O_TRUNC with the Open + // call which is more efficient for cmount. However + // it does not work with cgofuse on Windows with + // WinFSP so cmount must work with or without it. + "-o", "atomic_o_trunc", + "--FileSystemName=rclone", + } + + fsys := cmount.NewFS(f) + host := fuse.NewFileSystemHost(fsys) + + // Serve the mount point in the background returning error to errChan + r.unmountRes = make(chan error, 1) + go func() { + var err error + ok := host.Mount(r.mntDir, options) + if !ok { + err = errors.New("mount failed") + } + r.unmountRes <- err + }() + + // unmount + r.unmountFn = func() error { + // Shutdown the VFS + fsys.VFS.Shutdown() + if host.Unmount() { + if !waitFor(func() bool { + _, err := os.Stat(r.mntDir) + return err != nil + }) { + t.Fatalf("mountpoint %q didn't disappear after unmount - continuing anyway", r.mntDir) + } + return nil + } + return errors.New("host unmount failed") + } + + // Wait for the filesystem to become ready, checking the file + // system didn't blow up before starting + select { + case err := <-r.unmountRes: + require.NoError(t, err) + case <-time.After(time.Second * 3): + } + + // Wait for the mount point to be available on Windows + // On Windows the Init signal comes slightly before the mount is ready + if !waitFor(func() bool { + _, err := os.Stat(r.mntDir) + return err == nil + }) { + t.Errorf("mountpoint %q didn't became available on mount", r.mntDir) + } + + r.vfs = fsys.VFS + r.isMounted = true +} + +func (r *run) unmountFs(t *testing.T, f fs.Fs) { + // FIXME implement cmount + t.Skip("windows not supported yet") + var err error + + for i := 0; i < 4; i++ { + err = r.unmountFn() + if err != nil { + //log.Printf("signal to umount failed - retrying: %v", err) + time.Sleep(3 * time.Second) + continue + } + break + } + require.NoError(t, err) + err = <-r.unmountRes + require.NoError(t, err) + err = r.vfs.CleanUp() + require.NoError(t, err) + r.isMounted = false +} diff --git a/.rclone_repo/backend/cache/cache_test.go b/.rclone_repo/backend/cache/cache_test.go new file mode 100755 index 0000000..f0d29a2 --- /dev/null +++ b/.rclone_repo/backend/cache/cache_test.go @@ -0,0 +1,21 @@ +// Test Cache filesystem interface + +// +build !plan9 + +package cache_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/cache" + _ "github.com/ncw/rclone/backend/local" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestCache:", + NilObject: (*cache.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/cache/cache_unsupported.go b/.rclone_repo/backend/cache/cache_unsupported.go new file mode 100755 index 0000000..05a39fa --- /dev/null +++ b/.rclone_repo/backend/cache/cache_unsupported.go @@ -0,0 +1,6 @@ +// Build for cache for unsupported platforms to stop go complaining +// about "no buildable Go source files " + +// +build plan9 + +package cache diff --git a/.rclone_repo/backend/cache/cache_upload_test.go b/.rclone_repo/backend/cache/cache_upload_test.go new file mode 100755 index 0000000..c81a370 --- /dev/null +++ b/.rclone_repo/backend/cache/cache_upload_test.go @@ -0,0 +1,455 @@ +// +build !plan9 + +package cache_test + +import ( + "math/rand" + "os" + "path" + "strconv" + "testing" + "time" + + "fmt" + + "github.com/ncw/rclone/backend/cache" + _ "github.com/ncw/rclone/backend/drive" + "github.com/ncw/rclone/fs" + "github.com/stretchr/testify/require" +) + +func TestInternalUploadTempDirCreated(t *testing.T) { + id := fmt.Sprintf("tiutdc%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, + nil, + map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id)}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + _, err := os.Stat(path.Join(runInstance.tmpUploadDir, id)) + require.NoError(t, err) +} + +func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltDb *cache.Persistent) { + // create some rand test data + testSize := int64(524288000) + testReader := runInstance.randomReader(t, testSize) + bu := runInstance.listenForBackgroundUpload(t, rootFs, "one") + runInstance.writeRemoteReader(t, rootFs, "one", testReader) + // validate that it exists in temp fs + ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one"))) + require.NoError(t, err) + + if runInstance.rootIsCrypt { + require.Equal(t, int64(524416032), ti.Size()) + } else { + require.Equal(t, testSize, ti.Size()) + } + de1, err := runInstance.list(t, rootFs, "") + require.NoError(t, err) + require.Len(t, de1, 1) + + runInstance.completeBackgroundUpload(t, "one", bu) + // check if it was removed from temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one"))) + require.True(t, os.IsNotExist(err)) + + // check if it can be read + data2, err := runInstance.readDataFromRemote(t, rootFs, "one", 0, int64(1024), false) + require.NoError(t, err) + require.Len(t, data2, 1024) +} + +func TestInternalUploadQueueOneFileNoRest(t *testing.T) { + id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "0s"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + testInternalUploadQueueOneFile(t, id, rootFs, boltDb) +} + +func TestInternalUploadQueueOneFileWithRest(t *testing.T) { + id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1m"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + testInternalUploadQueueOneFile(t, id, rootFs, boltDb) +} + +func TestInternalUploadMoveExistingFile(t *testing.T) { + id := fmt.Sprintf("tiumef%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "3s"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + err := rootFs.Mkdir("one") + require.NoError(t, err) + err = rootFs.Mkdir("one/test") + require.NoError(t, err) + err = rootFs.Mkdir("second") + require.NoError(t, err) + + // create some rand test data + testSize := int64(10485760) + testReader := runInstance.randomReader(t, testSize) + runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader) + runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin") + + de1, err := runInstance.list(t, rootFs, "one/test") + require.NoError(t, err) + require.Len(t, de1, 1) + + time.Sleep(time.Second * 5) + //_ = os.Remove(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test"))) + //require.NoError(t, err) + + err = runInstance.dirMove(t, rootFs, "one/test", "second/test") + require.NoError(t, err) + + // check if it can be read + de1, err = runInstance.list(t, rootFs, "second/test") + require.NoError(t, err) + require.Len(t, de1, 1) +} + +func TestInternalUploadTempPathCleaned(t *testing.T) { + id := fmt.Sprintf("tiutpc%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + err := rootFs.Mkdir("one") + require.NoError(t, err) + err = rootFs.Mkdir("one/test") + require.NoError(t, err) + err = rootFs.Mkdir("second") + require.NoError(t, err) + + // create some rand test data + testSize := int64(1048576) + testReader := runInstance.randomReader(t, testSize) + testReader2 := runInstance.randomReader(t, testSize) + runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader) + runInstance.writeObjectReader(t, rootFs, "second/data.bin", testReader2) + + runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin") + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test"))) + require.True(t, os.IsNotExist(err)) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one"))) + require.True(t, os.IsNotExist(err)) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second"))) + require.False(t, os.IsNotExist(err)) + + runInstance.completeAllBackgroundUploads(t, rootFs, "second/data.bin") + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/data.bin"))) + require.True(t, os.IsNotExist(err)) + + de1, err := runInstance.list(t, rootFs, "one/test") + require.NoError(t, err) + require.Len(t, de1, 1) + + // check if it can be read + de1, err = runInstance.list(t, rootFs, "second") + require.NoError(t, err) + require.Len(t, de1, 1) +} + +func TestInternalUploadQueueMoreFiles(t *testing.T) { + id := fmt.Sprintf("tiuqmf%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1s"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + err := rootFs.Mkdir("test") + require.NoError(t, err) + minSize := 5242880 + maxSize := 10485760 + totalFiles := 10 + rand.Seed(time.Now().Unix()) + + lastFile := "" + for i := 0; i < totalFiles; i++ { + size := int64(rand.Intn(maxSize-minSize) + minSize) + testReader := runInstance.randomReader(t, size) + remote := "test/" + strconv.Itoa(i) + ".bin" + runInstance.writeRemoteReader(t, rootFs, remote, testReader) + + // validate that it exists in temp fs + ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, remote))) + require.NoError(t, err) + require.Equal(t, size, runInstance.cleanSize(t, ti.Size())) + + if runInstance.wrappedIsExternal && i < totalFiles-1 { + time.Sleep(time.Second * 3) + } + lastFile = remote + } + + // check if cache lists all files, likely temp upload didn't finish yet + de1, err := runInstance.list(t, rootFs, "test") + require.NoError(t, err) + require.Len(t, de1, totalFiles) + + // wait for background uploader to do its thing + runInstance.completeAllBackgroundUploads(t, rootFs, lastFile) + + // retry until we have no more temp files and fail if they don't go down to 0 + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test"))) + require.True(t, os.IsNotExist(err)) + + // check if cache lists all files + de1, err = runInstance.list(t, rootFs, "test") + require.NoError(t, err) + require.Len(t, de1, totalFiles) +} + +func TestInternalUploadTempFileOperations(t *testing.T) { + id := "tiutfo" + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + boltDb.PurgeTempUploads() + + // create some rand test data + runInstance.mkdir(t, rootFs, "test") + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // check if it can be read + data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false) + require.NoError(t, err) + require.Equal(t, []byte("one content"), data1) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + + // test DirMove - allowed + err = runInstance.dirMove(t, rootFs, "test", "second") + if err != errNotSupported { + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.Error(t, err) + _, err = rootFs.NewObject("second/one") + require.NoError(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.Error(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one"))) + require.NoError(t, err) + _, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one"))) + require.Error(t, err) + var started bool + started, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "second/one"))) + require.NoError(t, err) + require.False(t, started) + runInstance.mkdir(t, rootFs, "test") + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + } + + // test Rmdir - allowed + err = runInstance.rm(t, rootFs, "test") + require.Error(t, err) + require.Contains(t, err.Error(), "directory not empty") + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one"))) + require.False(t, started) + require.NoError(t, err) + + // test Move/Rename -- allowed + err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second")) + if err != errNotSupported { + require.NoError(t, err) + // try to read from it + _, err = rootFs.NewObject("test/one") + require.Error(t, err) + _, err = rootFs.NewObject("test/second") + require.NoError(t, err) + data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false) + require.NoError(t, err) + require.Equal(t, []byte("one content"), data2) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.Error(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second"))) + require.NoError(t, err) + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + } + + // test Copy -- allowed + err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third")) + if err != errNotSupported { + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/third") + require.NoError(t, err) + data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false) + require.NoError(t, err) + require.Equal(t, []byte("one content"), data2) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third"))) + require.NoError(t, err) + } + + // test Remove -- allowed + err = runInstance.rm(t, rootFs, "test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.Error(t, err) + // validate that it doesn't exist in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.Error(t, err) + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // test Update -- allowed + firstModTime, err := runInstance.modTime(t, rootFs, "test/one") + require.NoError(t, err) + err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated") + require.NoError(t, err) + obj2, err := rootFs.NewObject("test/one") + require.NoError(t, err) + data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false) + require.Equal(t, "one content updated", string(data2)) + tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + if runInstance.rootIsCrypt { + require.Equal(t, int64(67), tmpInfo.Size()) + } else { + require.Equal(t, int64(len(data2)), tmpInfo.Size()) + } + + // test SetModTime -- allowed + secondModTime, err := runInstance.modTime(t, rootFs, "test/one") + require.NoError(t, err) + require.NotEqual(t, secondModTime, firstModTime) + require.NotEqual(t, time.Time{}, firstModTime) + require.NotEqual(t, time.Time{}, secondModTime) +} + +func TestInternalUploadUploadingFileOperations(t *testing.T) { + id := "tiuufo" + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + boltDb.PurgeTempUploads() + + // create some rand test data + runInstance.mkdir(t, rootFs, "test") + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // check if it can be read + data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false) + require.NoError(t, err) + require.Equal(t, []byte("one content"), data1) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + + err = boltDb.SetPendingUploadToStarted(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "test/one"))) + require.NoError(t, err) + + // test DirMove + err = runInstance.dirMove(t, rootFs, "test", "second") + if err != errNotSupported { + require.Error(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one"))) + require.Error(t, err) + } + + // test Rmdir + err = runInstance.rm(t, rootFs, "test") + require.Error(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it doesn't exist in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + + // test Move/Rename + err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second")) + if err != errNotSupported { + require.Error(t, err) + // try to read from it + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/second") + require.Error(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second"))) + require.Error(t, err) + } + + // test Copy -- allowed + err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third")) + if err != errNotSupported { + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/third") + require.NoError(t, err) + data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false) + require.NoError(t, err) + require.Equal(t, []byte("one content"), data2) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third"))) + require.NoError(t, err) + } + + // test Remove + err = runInstance.rm(t, rootFs, "test/one") + require.Error(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it doesn't exist in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // test Update - this seems to work. Why? FIXME + //firstModTime, err := runInstance.modTime(t, rootFs, "test/one") + //require.NoError(t, err) + //err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated", func() { + // data2 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len("one content updated")), true) + // require.Equal(t, "one content", string(data2)) + // + // tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + // require.NoError(t, err) + // if runInstance.rootIsCrypt { + // require.Equal(t, int64(67), tmpInfo.Size()) + // } else { + // require.Equal(t, int64(len(data2)), tmpInfo.Size()) + // } + //}) + //require.Error(t, err) + + // test SetModTime -- seems to work cause of previous + //secondModTime, err := runInstance.modTime(t, rootFs, "test/one") + //require.NoError(t, err) + //require.Equal(t, secondModTime, firstModTime) + //require.NotEqual(t, time.Time{}, firstModTime) + //require.NotEqual(t, time.Time{}, secondModTime) +} diff --git a/.rclone_repo/backend/cache/cache_upload_test.go.orig b/.rclone_repo/backend/cache/cache_upload_test.go.orig new file mode 100755 index 0000000..e6072f3 --- /dev/null +++ b/.rclone_repo/backend/cache/cache_upload_test.go.orig @@ -0,0 +1,455 @@ +// +build !plan9 + +package cache_test + +import ( + "math/rand" + "os" + "path" + "strconv" + "testing" + "time" + + "fmt" + + "github.com/ncw/rclone/backend/cache" + _ "github.com/ncw/rclone/backend/drive" + "github.com/ncw/rclone/fs" + "github.com/stretchr/testify/require" +) + +func TestInternalUploadTempDirCreated(t *testing.T) { + id := fmt.Sprintf("tiutdc%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id)}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + _, err := os.Stat(path.Join(runInstance.tmpUploadDir, id)) + require.NoError(t, err) +} + +func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltDb *cache.Persistent) { + // create some rand test data + testSize := int64(524288000) + testReader := runInstance.randomReader(t, testSize) + bu := runInstance.listenForBackgroundUpload(t, rootFs, "one") + runInstance.writeRemoteReader(t, rootFs, "one", testReader) + // validate that it exists in temp fs + ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one"))) + require.NoError(t, err) + + if runInstance.rootIsCrypt { + require.Equal(t, int64(524416032), ti.Size()) + } else { + require.Equal(t, testSize, ti.Size()) + } + de1, err := runInstance.list(t, rootFs, "") + require.NoError(t, err) + require.Len(t, de1, 1) + + runInstance.completeBackgroundUpload(t, "one", bu) + // check if it was removed from temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one"))) + require.True(t, os.IsNotExist(err)) + + // check if it can be read + data2, err := runInstance.readDataFromRemote(t, rootFs, "one", 0, int64(1024), false) + require.NoError(t, err) + require.Len(t, data2, 1024) +} + +func TestInternalUploadQueueOneFileNoRest(t *testing.T) { + id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "0s"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + testInternalUploadQueueOneFile(t, id, rootFs, boltDb) +} + +func TestInternalUploadQueueOneFileWithRest(t *testing.T) { + id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1m"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + testInternalUploadQueueOneFile(t, id, rootFs, boltDb) +} + +func TestInternalUploadMoveExistingFile(t *testing.T) { + id := fmt.Sprintf("tiumef%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "3s"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + err := rootFs.Mkdir("one") + require.NoError(t, err) + err = rootFs.Mkdir("one/test") + require.NoError(t, err) + err = rootFs.Mkdir("second") + require.NoError(t, err) + + // create some rand test data + testSize := int64(10485760) + testReader := runInstance.randomReader(t, testSize) + runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader) + runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin") + + de1, err := runInstance.list(t, rootFs, "one/test") + require.NoError(t, err) + require.Len(t, de1, 1) + + time.Sleep(time.Second * 5) + //_ = os.Remove(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test"))) + //require.NoError(t, err) + + err = runInstance.dirMove(t, rootFs, "one/test", "second/test") + require.NoError(t, err) + + // check if it can be read + de1, err = runInstance.list(t, rootFs, "second/test") + require.NoError(t, err) + require.Len(t, de1, 1) +} + +func TestInternalUploadTempPathCleaned(t *testing.T) { + id := fmt.Sprintf("tiutpc%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + err := rootFs.Mkdir("one") + require.NoError(t, err) + err = rootFs.Mkdir("one/test") + require.NoError(t, err) + err = rootFs.Mkdir("second") + require.NoError(t, err) + + // create some rand test data + testSize := int64(1048576) + testReader := runInstance.randomReader(t, testSize) + testReader2 := runInstance.randomReader(t, testSize) + runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader) + runInstance.writeObjectReader(t, rootFs, "second/data.bin", testReader2) + + runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin") + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test"))) + require.True(t, os.IsNotExist(err)) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one"))) + require.True(t, os.IsNotExist(err)) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second"))) + require.False(t, os.IsNotExist(err)) + + runInstance.completeAllBackgroundUploads(t, rootFs, "second/data.bin") + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/data.bin"))) + require.True(t, os.IsNotExist(err)) + + de1, err := runInstance.list(t, rootFs, "one/test") + require.NoError(t, err) + require.Len(t, de1, 1) + + // check if it can be read + de1, err = runInstance.list(t, rootFs, "second") + require.NoError(t, err) + require.Len(t, de1, 1) +} + +func TestInternalUploadQueueMoreFiles(t *testing.T) { + id := fmt.Sprintf("tiuqmf%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1s"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + err := rootFs.Mkdir("test") + require.NoError(t, err) + minSize := 5242880 + maxSize := 10485760 + totalFiles := 10 + rand.Seed(time.Now().Unix()) + + lastFile := "" + for i := 0; i < totalFiles; i++ { + size := int64(rand.Intn(maxSize-minSize) + minSize) + testReader := runInstance.randomReader(t, size) + remote := "test/" + strconv.Itoa(i) + ".bin" + runInstance.writeRemoteReader(t, rootFs, remote, testReader) + + // validate that it exists in temp fs + ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, remote))) + require.NoError(t, err) + require.Equal(t, size, runInstance.cleanSize(t, ti.Size())) + + if runInstance.wrappedIsExternal && i < totalFiles-1 { + time.Sleep(time.Second * 3) + } + lastFile = remote + } + + // check if cache lists all files, likely temp upload didn't finish yet + de1, err := runInstance.list(t, rootFs, "test") + require.NoError(t, err) + require.Len(t, de1, totalFiles) + + // wait for background uploader to do its thing + runInstance.completeAllBackgroundUploads(t, rootFs, lastFile) + + // retry until we have no more temp files and fail if they don't go down to 0 + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test"))) + require.True(t, os.IsNotExist(err)) + + // check if cache lists all files + de1, err = runInstance.list(t, rootFs, "test") + require.NoError(t, err) + require.Len(t, de1, totalFiles) +} + +func TestInternalUploadTempFileOperations(t *testing.T) { + id := "tiutfo" + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + boltDb.PurgeTempUploads() + + // create some rand test data + runInstance.mkdir(t, rootFs, "test") + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // check if it can be read + data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false) + require.NoError(t, err) + require.Equal(t, []byte("one content"), data1) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + + // test DirMove - allowed + err = runInstance.dirMove(t, rootFs, "test", "second") + if err != errNotSupported { + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.Error(t, err) + _, err = rootFs.NewObject("second/one") + require.NoError(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.Error(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one"))) + require.NoError(t, err) + _, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one"))) + require.Error(t, err) + var started bool + started, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "second/one"))) + require.NoError(t, err) + require.False(t, started) + runInstance.mkdir(t, rootFs, "test") + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + } + + // test Rmdir - allowed + err = runInstance.rm(t, rootFs, "test") + require.Error(t, err) + require.Contains(t, err.Error(), "directory not empty") + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one"))) + require.False(t, started) + require.NoError(t, err) + + // test Move/Rename -- allowed + err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second")) + if err != errNotSupported { + require.NoError(t, err) + // try to read from it + _, err = rootFs.NewObject("test/one") + require.Error(t, err) + _, err = rootFs.NewObject("test/second") + require.NoError(t, err) + data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false) + require.NoError(t, err) + require.Equal(t, []byte("one content"), data2) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.Error(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second"))) + require.NoError(t, err) + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + } + + // test Copy -- allowed + err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third")) + if err != errNotSupported { + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/third") + require.NoError(t, err) + data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false) + require.NoError(t, err) + require.Equal(t, []byte("one content"), data2) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third"))) + require.NoError(t, err) + } + + // test Remove -- allowed + err = runInstance.rm(t, rootFs, "test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.Error(t, err) + // validate that it doesn't exist in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.Error(t, err) + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // test Update -- allowed + firstModTime, err := runInstance.modTime(t, rootFs, "test/one") + require.NoError(t, err) + err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated") + require.NoError(t, err) + obj2, err := rootFs.NewObject("test/one") + require.NoError(t, err) + data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false) + require.Equal(t, "one content updated", string(data2)) + tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + if runInstance.rootIsCrypt { + require.Equal(t, int64(67), tmpInfo.Size()) + } else { + require.Equal(t, int64(len(data2)), tmpInfo.Size()) + } + + // test SetModTime -- allowed + secondModTime, err := runInstance.modTime(t, rootFs, "test/one") + require.NoError(t, err) + require.NotEqual(t, secondModTime, firstModTime) + require.NotEqual(t, time.Time{}, firstModTime) + require.NotEqual(t, time.Time{}, secondModTime) +} + +func TestInternalUploadUploadingFileOperations(t *testing.T) { + id := "tiuufo" + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + boltDb.PurgeTempUploads() + + // create some rand test data + runInstance.mkdir(t, rootFs, "test") + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // check if it can be read + data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false) + require.NoError(t, err) + require.Equal(t, []byte("one content"), data1) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + + err = boltDb.SetPendingUploadToStarted(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "test/one"))) + require.NoError(t, err) + + // test DirMove + err = runInstance.dirMove(t, rootFs, "test", "second") + if err != errNotSupported { + require.Error(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one"))) + require.Error(t, err) + } + + // test Rmdir + err = runInstance.rm(t, rootFs, "test") + require.Error(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it doesn't exist in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + + // test Move/Rename + err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second")) + if err != errNotSupported { + require.Error(t, err) + // try to read from it + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/second") + require.Error(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second"))) + require.Error(t, err) + } + + // test Copy -- allowed + err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third")) + if err != errNotSupported { + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/third") + require.NoError(t, err) + data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false) + require.NoError(t, err) + require.Equal(t, []byte("one content"), data2) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third"))) + require.NoError(t, err) + } + + // test Remove + err = runInstance.rm(t, rootFs, "test/one") + require.Error(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it doesn't exist in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // test Update - this seems to work. Why? FIXME + //firstModTime, err := runInstance.modTime(t, rootFs, "test/one") + //require.NoError(t, err) + //err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated", func() { + // data2 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len("one content updated")), true) + // require.Equal(t, "one content", string(data2)) + // + // tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + // require.NoError(t, err) + // if runInstance.rootIsCrypt { + // require.Equal(t, int64(67), tmpInfo.Size()) + // } else { + // require.Equal(t, int64(len(data2)), tmpInfo.Size()) + // } + //}) + //require.Error(t, err) + + // test SetModTime -- seems to work cause of previous + //secondModTime, err := runInstance.modTime(t, rootFs, "test/one") + //require.NoError(t, err) + //require.Equal(t, secondModTime, firstModTime) + //require.NotEqual(t, time.Time{}, firstModTime) + //require.NotEqual(t, time.Time{}, secondModTime) +} diff --git a/.rclone_repo/backend/cache/cache_upload_test.go.rej b/.rclone_repo/backend/cache/cache_upload_test.go.rej new file mode 100755 index 0000000..a84a3d8 --- /dev/null +++ b/.rclone_repo/backend/cache/cache_upload_test.go.rej @@ -0,0 +1,12 @@ +--- cache_upload_test.go ++++ cache_upload_test.go +@@ -1500,9 +1469,6 @@ func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) { + } + r.tempFiles = nil + debug.FreeOSMemory() +- for k, v := range r.runDefaultFlagMap { +- _ = flag.Set(k, v) +- } + } + + func (r *run) randomBytes(t *testing.T, size int64) []byte { diff --git a/.rclone_repo/backend/cache/directory.go b/.rclone_repo/backend/cache/directory.go new file mode 100755 index 0000000..8636127 --- /dev/null +++ b/.rclone_repo/backend/cache/directory.go @@ -0,0 +1,138 @@ +// +build !plan9 + +package cache + +import ( + "time" + + "path" + + "github.com/ncw/rclone/fs" +) + +// Directory is a generic dir that stores basic information about it +type Directory struct { + Directory fs.Directory `json:"-"` // can be nil + + CacheFs *Fs `json:"-"` // cache fs + Name string `json:"name"` // name of the directory + Dir string `json:"dir"` // abs path of the directory + CacheModTime int64 `json:"modTime"` // modification or creation time - IsZero for unknown + CacheSize int64 `json:"size"` // size of directory and contents or -1 if unknown + + CacheItems int64 `json:"items"` // number of objects or -1 for unknown + CacheType string `json:"cacheType"` // object type + CacheTs *time.Time `json:",omitempty"` +} + +// NewDirectory builds an empty dir which will be used to unmarshal data in it +func NewDirectory(f *Fs, remote string) *Directory { + cd := ShallowDirectory(f, remote) + t := time.Now() + cd.CacheTs = &t + + return cd +} + +// ShallowDirectory builds an empty dir which will be used to unmarshal data in it +func ShallowDirectory(f *Fs, remote string) *Directory { + var cd *Directory + fullRemote := cleanPath(path.Join(f.Root(), remote)) + + // build a new one + dir := cleanPath(path.Dir(fullRemote)) + name := cleanPath(path.Base(fullRemote)) + cd = &Directory{ + CacheFs: f, + Name: name, + Dir: dir, + CacheModTime: time.Now().UnixNano(), + CacheSize: 0, + CacheItems: 0, + CacheType: "Directory", + } + + return cd +} + +// DirectoryFromOriginal builds one from a generic fs.Directory +func DirectoryFromOriginal(f *Fs, d fs.Directory) *Directory { + var cd *Directory + fullRemote := path.Join(f.Root(), d.Remote()) + + dir := cleanPath(path.Dir(fullRemote)) + name := cleanPath(path.Base(fullRemote)) + t := time.Now() + cd = &Directory{ + Directory: d, + CacheFs: f, + Name: name, + Dir: dir, + CacheModTime: d.ModTime().UnixNano(), + CacheSize: d.Size(), + CacheItems: d.Items(), + CacheType: "Directory", + CacheTs: &t, + } + + return cd +} + +// Fs returns its FS info +func (d *Directory) Fs() fs.Info { + return d.CacheFs +} + +// String returns a human friendly name for this object +func (d *Directory) String() string { + if d == nil { + return "" + } + return d.Remote() +} + +// Remote returns the remote path +func (d *Directory) Remote() string { + return d.CacheFs.cleanRootFromPath(d.abs()) +} + +// abs returns the absolute path to the dir +func (d *Directory) abs() string { + return cleanPath(path.Join(d.Dir, d.Name)) +} + +// parentRemote returns the absolute path parent remote +func (d *Directory) parentRemote() string { + absPath := d.abs() + if absPath == "" { + return "" + } + return cleanPath(path.Dir(absPath)) +} + +// ModTime returns the cached ModTime +func (d *Directory) ModTime() time.Time { + return time.Unix(0, d.CacheModTime) +} + +// Size returns the cached Size +func (d *Directory) Size() int64 { + return d.CacheSize +} + +// Items returns the cached Items +func (d *Directory) Items() int64 { + return d.CacheItems +} + +// ID returns the ID of the cached directory if known +func (d *Directory) ID() string { + if d.Directory == nil { + return "" + } + return d.Directory.ID() +} + +var ( + _ fs.Directory = (*Directory)(nil) +) diff --git a/.rclone_repo/backend/cache/handle.go b/.rclone_repo/backend/cache/handle.go new file mode 100755 index 0000000..f608299 --- /dev/null +++ b/.rclone_repo/backend/cache/handle.go @@ -0,0 +1,668 @@ +// +build !plan9 + +package cache + +import ( + "fmt" + "io" + "sync" + "time" + + "path" + "runtime" + "strings" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/operations" + "github.com/pkg/errors" +) + +var uploaderMap = make(map[string]*backgroundWriter) +var uploaderMapMx sync.Mutex + +// initBackgroundUploader returns a single instance +func initBackgroundUploader(fs *Fs) (*backgroundWriter, error) { + // write lock to create one + uploaderMapMx.Lock() + defer uploaderMapMx.Unlock() + if b, ok := uploaderMap[fs.String()]; ok { + // if it was already started we close it so that it can be started again + if b.running { + b.close() + } else { + return b, nil + } + } + + bb := newBackgroundWriter(fs) + uploaderMap[fs.String()] = bb + return uploaderMap[fs.String()], nil +} + +// Handle is managing the read/write/seek operations on an open handle +type Handle struct { + cachedObject *Object + cfs *Fs + memory *Memory + preloadQueue chan int64 + preloadOffset int64 + offset int64 + seenOffsets map[int64]bool + mu sync.Mutex + confirmReading chan bool + + UseMemory bool + workers []*worker + closed bool + reading bool +} + +// NewObjectHandle returns a new Handle for an existing Object +func NewObjectHandle(o *Object, cfs *Fs) *Handle { + r := &Handle{ + cachedObject: o, + cfs: cfs, + offset: 0, + preloadOffset: -1, // -1 to trigger the first preload + + UseMemory: !cfs.opt.ChunkNoMemory, + reading: false, + } + r.seenOffsets = make(map[int64]bool) + r.memory = NewMemory(-1) + + // create a larger buffer to queue up requests + r.preloadQueue = make(chan int64, r.cfs.opt.TotalWorkers*10) + r.confirmReading = make(chan bool) + r.startReadWorkers() + return r +} + +// cacheFs is a convenience method to get the parent cache FS of the object's manager +func (r *Handle) cacheFs() *Fs { + return r.cfs +} + +// storage is a convenience method to get the persistent storage of the object's manager +func (r *Handle) storage() *Persistent { + return r.cacheFs().cache +} + +// String representation of this reader +func (r *Handle) String() string { + return r.cachedObject.abs() +} + +// startReadWorkers will start the worker pool +func (r *Handle) startReadWorkers() { + if r.hasAtLeastOneWorker() { + return + } + totalWorkers := r.cacheFs().opt.TotalWorkers + + if r.cacheFs().plexConnector.isConfigured() { + if !r.cacheFs().plexConnector.isConnected() { + err := r.cacheFs().plexConnector.authenticate() + if err != nil { + fs.Errorf(r, "failed to authenticate to Plex: %v", err) + } + } + if r.cacheFs().plexConnector.isConnected() { + totalWorkers = 1 + } + } + + r.scaleWorkers(totalWorkers) +} + +// scaleOutWorkers will increase the worker pool count by the provided amount +func (r *Handle) scaleWorkers(desired int) { + current := len(r.workers) + if current == desired { + return + } + if current > desired { + // scale in gracefully + for i := 0; i < current-desired; i++ { + r.preloadQueue <- -1 + } + } else { + // scale out + for i := 0; i < desired-current; i++ { + w := &worker{ + r: r, + ch: r.preloadQueue, + id: current + i, + } + go w.run() + + r.workers = append(r.workers, w) + } + } + // ignore first scale out from 0 + if current != 0 { + fs.Debugf(r, "scale workers to %v", desired) + } +} + +func (r *Handle) confirmExternalReading() { + // if we have a max value of workers + // then we skip this step + if len(r.workers) > 1 || + !r.cacheFs().plexConnector.isConfigured() { + return + } + if !r.cacheFs().plexConnector.isPlaying(r.cachedObject) { + return + } + fs.Infof(r, "confirmed reading by external reader") + r.scaleWorkers(r.cacheFs().opt.TotalWorkers) +} + +// queueOffset will send an offset to the workers if it's different from the last one +func (r *Handle) queueOffset(offset int64) { + if offset != r.preloadOffset { + // clean past in-memory chunks + if r.UseMemory { + go r.memory.CleanChunksByNeed(offset) + } + r.confirmExternalReading() + r.preloadOffset = offset + + // clear the past seen chunks + // they will remain in our persistent storage but will be removed from transient + // so they need to be picked up by a worker + for k := range r.seenOffsets { + if k < offset { + r.seenOffsets[k] = false + } + } + + for i := 0; i < len(r.workers); i++ { + o := r.preloadOffset + int64(r.cacheFs().opt.ChunkSize)*int64(i) + if o < 0 || o >= r.cachedObject.Size() { + continue + } + if v, ok := r.seenOffsets[o]; ok && v { + continue + } + + r.seenOffsets[o] = true + r.preloadQueue <- o + } + } +} + +func (r *Handle) hasAtLeastOneWorker() bool { + oneWorker := false + for i := 0; i < len(r.workers); i++ { + if r.workers[i].isRunning() { + oneWorker = true + } + } + return oneWorker +} + +// getChunk is called by the FS to retrieve a specific chunk of known start and size from where it can find it +// it can be from transient or persistent cache +// it will also build the chunk from the cache's specific chunk boundaries and build the final desired chunk in a buffer +func (r *Handle) getChunk(chunkStart int64) ([]byte, error) { + var data []byte + var err error + + // we calculate the modulus of the requested offset with the size of a chunk + offset := chunkStart % int64(r.cacheFs().opt.ChunkSize) + + // we align the start offset of the first chunk to a likely chunk in the storage + chunkStart = chunkStart - offset + r.queueOffset(chunkStart) + found := false + + if r.UseMemory { + data, err = r.memory.GetChunk(r.cachedObject, chunkStart) + if err == nil { + found = true + } + } + + if !found { + // we're gonna give the workers a chance to pickup the chunk + // and retry a couple of times + for i := 0; i < r.cacheFs().opt.ReadRetries*8; i++ { + data, err = r.storage().GetChunk(r.cachedObject, chunkStart) + if err == nil { + found = true + break + } + + fs.Debugf(r, "%v: chunk retry storage: %v", chunkStart, i) + time.Sleep(time.Millisecond * 500) + } + } + + // not found in ram or + // the worker didn't managed to download the chunk in time so we abort and close the stream + if err != nil || len(data) == 0 || !found { + if !r.hasAtLeastOneWorker() { + fs.Errorf(r, "out of workers") + return nil, io.ErrUnexpectedEOF + } + + return nil, errors.Errorf("chunk not found %v", chunkStart) + } + + // first chunk will be aligned with the start + if offset > 0 { + if offset > int64(len(data)) { + fs.Errorf(r, "unexpected conditions during reading. current position: %v, current chunk position: %v, current chunk size: %v, offset: %v, chunk size: %v, file size: %v", + r.offset, chunkStart, len(data), offset, r.cacheFs().opt.ChunkSize, r.cachedObject.Size()) + return nil, io.ErrUnexpectedEOF + } + data = data[int(offset):] + } + + return data, nil +} + +// Read a chunk from storage or len(p) +func (r *Handle) Read(p []byte) (n int, err error) { + r.mu.Lock() + defer r.mu.Unlock() + var buf []byte + + // first reading + if !r.reading { + r.reading = true + } + // reached EOF + if r.offset >= r.cachedObject.Size() { + return 0, io.EOF + } + currentOffset := r.offset + buf, err = r.getChunk(currentOffset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + fs.Errorf(r, "(%v/%v) error (%v) response", currentOffset, r.cachedObject.Size(), err) + } + if len(buf) == 0 && err != io.ErrUnexpectedEOF { + return 0, io.EOF + } + readSize := copy(p, buf) + newOffset := currentOffset + int64(readSize) + r.offset = newOffset + + return readSize, err +} + +// Close will tell the workers to stop +func (r *Handle) Close() error { + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return errors.New("file already closed") + } + + close(r.preloadQueue) + r.closed = true + // wait for workers to complete their jobs before returning + waitCount := 3 + for i := 0; i < len(r.workers); i++ { + waitIdx := 0 + for r.workers[i].isRunning() && waitIdx < waitCount { + time.Sleep(time.Second) + waitIdx++ + } + } + r.memory.db.Flush() + + fs.Debugf(r, "cache reader closed %v", r.offset) + return nil +} + +// Seek will move the current offset based on whence and instruct the workers to move there too +func (r *Handle) Seek(offset int64, whence int) (int64, error) { + r.mu.Lock() + defer r.mu.Unlock() + + var err error + switch whence { + case io.SeekStart: + fs.Debugf(r, "moving offset set from %v to %v", r.offset, offset) + r.offset = offset + case io.SeekCurrent: + fs.Debugf(r, "moving offset cur from %v to %v", r.offset, r.offset+offset) + r.offset += offset + case io.SeekEnd: + fs.Debugf(r, "moving offset end (%v) from %v to %v", r.cachedObject.Size(), r.offset, r.cachedObject.Size()+offset) + r.offset = r.cachedObject.Size() + offset + default: + err = errors.Errorf("cache: unimplemented seek whence %v", whence) + } + + chunkStart := r.offset - (r.offset % int64(r.cacheFs().opt.ChunkSize)) + if chunkStart >= int64(r.cacheFs().opt.ChunkSize) { + chunkStart = chunkStart - int64(r.cacheFs().opt.ChunkSize) + } + r.queueOffset(chunkStart) + + return r.offset, err +} + +type worker struct { + r *Handle + ch <-chan int64 + rc io.ReadCloser + id int + running bool + mu sync.Mutex +} + +// String is a representation of this worker +func (w *worker) String() string { + return fmt.Sprintf("worker-%v <%v>", w.id, w.r.cachedObject.Name) +} + +// reader will return a reader depending on the capabilities of the source reader: +// - if it supports seeking it will seek to the desired offset and return the same reader +// - if it doesn't support seeking it will close a possible existing one and open at the desired offset +// - if there's no reader associated with this worker, it will create one +func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error) { + var err error + r := w.rc + if w.rc == nil { + r, err = w.r.cacheFs().openRateLimited(func() (io.ReadCloser, error) { + return w.r.cachedObject.Object.Open(&fs.RangeOption{Start: offset, End: end - 1}) + }) + if err != nil { + return nil, err + } + return r, nil + } + + if !closeOpen { + if do, ok := r.(fs.RangeSeeker); ok { + _, err = do.RangeSeek(offset, io.SeekStart, end-offset) + return r, err + } else if do, ok := r.(io.Seeker); ok { + _, err = do.Seek(offset, io.SeekStart) + return r, err + } + } + + _ = w.rc.Close() + return w.r.cacheFs().openRateLimited(func() (io.ReadCloser, error) { + r, err = w.r.cachedObject.Object.Open(&fs.RangeOption{Start: offset, End: end - 1}) + if err != nil { + return nil, err + } + return r, nil + }) +} + +func (w *worker) isRunning() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.running +} + +func (w *worker) setRunning(f bool) { + w.mu.Lock() + defer w.mu.Unlock() + w.running = f +} + +// run is the main loop for the worker which receives offsets to preload +func (w *worker) run() { + var err error + var data []byte + defer w.setRunning(false) + defer func() { + if w.rc != nil { + _ = w.rc.Close() + w.setRunning(false) + } + }() + + for { + chunkStart, open := <-w.ch + w.setRunning(true) + if chunkStart < 0 || !open { + break + } + + // skip if it exists + if w.r.UseMemory { + if w.r.memory.HasChunk(w.r.cachedObject, chunkStart) { + continue + } + + // add it in ram if it's in the persistent storage + data, err = w.r.storage().GetChunk(w.r.cachedObject, chunkStart) + if err == nil { + err = w.r.memory.AddChunk(w.r.cachedObject.abs(), data, chunkStart) + if err != nil { + fs.Errorf(w, "failed caching chunk in ram %v: %v", chunkStart, err) + } else { + continue + } + } + } else { + if w.r.storage().HasChunk(w.r.cachedObject, chunkStart) { + continue + } + } + + chunkEnd := chunkStart + int64(w.r.cacheFs().opt.ChunkSize) + // TODO: Remove this comment if it proves to be reliable for #1896 + //if chunkEnd > w.r.cachedObject.Size() { + // chunkEnd = w.r.cachedObject.Size() + //} + + w.download(chunkStart, chunkEnd, 0) + } +} + +func (w *worker) download(chunkStart, chunkEnd int64, retry int) { + var err error + var data []byte + + // stop retries + if retry >= w.r.cacheFs().opt.ReadRetries { + return + } + // back-off between retries + if retry > 0 { + time.Sleep(time.Second * time.Duration(retry)) + } + + closeOpen := false + if retry > 0 { + closeOpen = true + } + w.rc, err = w.reader(chunkStart, chunkEnd, closeOpen) + // we seem to be getting only errors so we abort + if err != nil { + fs.Errorf(w, "object open failed %v: %v", chunkStart, err) + err = w.r.cachedObject.refreshFromSource(true) + if err != nil { + fs.Errorf(w, "%v", err) + } + w.download(chunkStart, chunkEnd, retry+1) + return + } + + data = make([]byte, chunkEnd-chunkStart) + var sourceRead int + sourceRead, err = io.ReadFull(w.rc, data) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + fs.Errorf(w, "failed to read chunk %v: %v", chunkStart, err) + err = w.r.cachedObject.refreshFromSource(true) + if err != nil { + fs.Errorf(w, "%v", err) + } + w.download(chunkStart, chunkEnd, retry+1) + return + } + data = data[:sourceRead] // reslice to remove extra garbage + if err == io.ErrUnexpectedEOF { + fs.Debugf(w, "partial downloaded chunk %v", fs.SizeSuffix(chunkStart)) + } else { + fs.Debugf(w, "downloaded chunk %v", chunkStart) + } + + if w.r.UseMemory { + err = w.r.memory.AddChunk(w.r.cachedObject.abs(), data, chunkStart) + if err != nil { + fs.Errorf(w, "failed caching chunk in ram %v: %v", chunkStart, err) + } + } + + err = w.r.storage().AddChunk(w.r.cachedObject.abs(), data, chunkStart) + if err != nil { + fs.Errorf(w, "failed caching chunk in storage %v: %v", chunkStart, err) + } +} + +const ( + // BackgroundUploadStarted is a state for a temp file that has started upload + BackgroundUploadStarted = iota + // BackgroundUploadCompleted is a state for a temp file that has completed upload + BackgroundUploadCompleted + // BackgroundUploadError is a state for a temp file that has an error upload + BackgroundUploadError +) + +// BackgroundUploadState is an entity that maps to an existing file which is stored on the temp fs +type BackgroundUploadState struct { + Remote string + Status int + Error error +} + +type backgroundWriter struct { + fs *Fs + stateCh chan int + running bool + notifyCh chan BackgroundUploadState + mu sync.Mutex +} + +func newBackgroundWriter(f *Fs) *backgroundWriter { + b := &backgroundWriter{ + fs: f, + stateCh: make(chan int), + notifyCh: make(chan BackgroundUploadState), + } + + return b +} + +func (b *backgroundWriter) close() { + b.stateCh <- 2 + b.mu.Lock() + defer b.mu.Unlock() + b.running = false + +} + +func (b *backgroundWriter) pause() { + b.stateCh <- 1 +} + +func (b *backgroundWriter) play() { + b.stateCh <- 0 +} + +func (b *backgroundWriter) isRunning() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.running +} + +func (b *backgroundWriter) notify(remote string, status int, err error) { + state := BackgroundUploadState{ + Remote: remote, + Status: status, + Error: err, + } + select { + case b.notifyCh <- state: + fs.Debugf(remote, "notified background upload state: %v", state.Status) + default: + } +} + +func (b *backgroundWriter) run() { + state := 0 + for { + b.mu.Lock() + b.running = true + b.mu.Unlock() + select { + case s := <-b.stateCh: + state = s + default: + // + } + switch state { + case 1: + runtime.Gosched() + time.Sleep(time.Millisecond * 500) + continue + case 2: + return + } + + absPath, err := b.fs.cache.getPendingUpload(b.fs.Root(), time.Duration(b.fs.opt.TempWaitTime)) + if err != nil || absPath == "" || !b.fs.isRootInPath(absPath) { + time.Sleep(time.Second) + continue + } + + remote := b.fs.cleanRootFromPath(absPath) + b.notify(remote, BackgroundUploadStarted, nil) + fs.Infof(remote, "background upload: started upload") + err = operations.MoveFile(b.fs.UnWrap(), b.fs.tempFs, remote, remote) + if err != nil { + b.notify(remote, BackgroundUploadError, err) + _ = b.fs.cache.rollbackPendingUpload(absPath) + fs.Errorf(remote, "background upload: %v", err) + continue + } + // clean empty dirs up to root + thisDir := cleanPath(path.Dir(remote)) + for thisDir != "" { + thisList, err := b.fs.tempFs.List(thisDir) + if err != nil { + break + } + if len(thisList) > 0 { + break + } + err = b.fs.tempFs.Rmdir(thisDir) + fs.Debugf(thisDir, "cleaned from temp path") + if err != nil { + break + } + thisDir = cleanPath(path.Dir(thisDir)) + } + fs.Infof(remote, "background upload: uploaded entry") + err = b.fs.cache.removePendingUpload(absPath) + if err != nil && !strings.Contains(err.Error(), "pending upload not found") { + fs.Errorf(remote, "background upload: %v", err) + } + parentCd := NewDirectory(b.fs, cleanPath(path.Dir(remote))) + err = b.fs.cache.ExpireDir(parentCd) + if err != nil { + fs.Errorf(parentCd, "background upload: cache expire error: %v", err) + } + b.fs.notifyChangeUpstream(remote, fs.EntryObject) + fs.Infof(remote, "finished background upload") + b.notify(remote, BackgroundUploadCompleted, nil) + } +} + +// Check the interfaces are satisfied +var ( + _ io.ReadCloser = (*Handle)(nil) + _ io.Seeker = (*Handle)(nil) +) diff --git a/.rclone_repo/backend/cache/object.go b/.rclone_repo/backend/cache/object.go new file mode 100755 index 0000000..c567a82 --- /dev/null +++ b/.rclone_repo/backend/cache/object.go @@ -0,0 +1,365 @@ +// +build !plan9 + +package cache + +import ( + "io" + "path" + "sync" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/readers" + "github.com/pkg/errors" +) + +const ( + objectInCache = "Object" + objectPendingUpload = "TempObject" +) + +// Object is a generic file like object that stores basic information about it +type Object struct { + fs.Object `json:"-"` + + ParentFs fs.Fs `json:"-"` // parent fs + CacheFs *Fs `json:"-"` // cache fs + Name string `json:"name"` // name of the directory + Dir string `json:"dir"` // abs path of the object + CacheModTime int64 `json:"modTime"` // modification or creation time - IsZero for unknown + CacheSize int64 `json:"size"` // size of directory and contents or -1 if unknown + CacheStorable bool `json:"storable"` // says whether this object can be stored + CacheType string `json:"cacheType"` + CacheTs time.Time `json:"cacheTs"` + CacheHashes map[hash.Type]string // all supported hashes cached + + refreshMutex sync.Mutex +} + +// NewObject builds one from a generic fs.Object +func NewObject(f *Fs, remote string) *Object { + fullRemote := path.Join(f.Root(), remote) + dir, name := path.Split(fullRemote) + + cacheType := objectInCache + parentFs := f.UnWrap() + if f.opt.TempWritePath != "" { + _, err := f.cache.SearchPendingUpload(fullRemote) + if err == nil { // queued for upload + cacheType = objectPendingUpload + parentFs = f.tempFs + fs.Debugf(fullRemote, "pending upload found") + } + } + + co := &Object{ + ParentFs: parentFs, + CacheFs: f, + Name: cleanPath(name), + Dir: cleanPath(dir), + CacheModTime: time.Now().UnixNano(), + CacheSize: 0, + CacheStorable: false, + CacheType: cacheType, + CacheTs: time.Now(), + } + return co +} + +// ObjectFromOriginal builds one from a generic fs.Object +func ObjectFromOriginal(f *Fs, o fs.Object) *Object { + var co *Object + fullRemote := cleanPath(path.Join(f.Root(), o.Remote())) + dir, name := path.Split(fullRemote) + + cacheType := objectInCache + parentFs := f.UnWrap() + if f.opt.TempWritePath != "" { + _, err := f.cache.SearchPendingUpload(fullRemote) + if err == nil { // queued for upload + cacheType = objectPendingUpload + parentFs = f.tempFs + fs.Debugf(fullRemote, "pending upload found") + } + } + + co = &Object{ + ParentFs: parentFs, + CacheFs: f, + Name: cleanPath(name), + Dir: cleanPath(dir), + CacheType: cacheType, + CacheTs: time.Now(), + } + co.updateData(o) + return co +} + +func (o *Object) updateData(source fs.Object) { + o.Object = source + o.CacheModTime = source.ModTime().UnixNano() + o.CacheSize = source.Size() + o.CacheStorable = source.Storable() + o.CacheTs = time.Now() + o.CacheHashes = make(map[hash.Type]string) +} + +// Fs returns its FS info +func (o *Object) Fs() fs.Info { + return o.CacheFs +} + +// String returns a human friendly name for this object +func (o *Object) String() string { + if o == nil { + return "" + } + return o.Remote() +} + +// Remote returns the remote path +func (o *Object) Remote() string { + p := path.Join(o.Dir, o.Name) + return o.CacheFs.cleanRootFromPath(p) +} + +// abs returns the absolute path to the object +func (o *Object) abs() string { + return path.Join(o.Dir, o.Name) +} + +// ModTime returns the cached ModTime +func (o *Object) ModTime() time.Time { + _ = o.refresh() + return time.Unix(0, o.CacheModTime) +} + +// Size returns the cached Size +func (o *Object) Size() int64 { + _ = o.refresh() + return o.CacheSize +} + +// Storable returns the cached Storable +func (o *Object) Storable() bool { + _ = o.refresh() + return o.CacheStorable +} + +// refresh will check if the object info is expired and request the info from source if it is +// all these conditions must be true to ignore a refresh +// 1. cache ts didn't expire yet +// 2. is not pending a notification from the wrapped fs +func (o *Object) refresh() error { + isNotified := o.CacheFs.isNotifiedRemote(o.Remote()) + isExpired := time.Now().After(o.CacheTs.Add(time.Duration(o.CacheFs.opt.InfoAge))) + if !isExpired && !isNotified { + return nil + } + + return o.refreshFromSource(true) +} + +// refreshFromSource requests the original FS for the object in case it comes from a cached entry +func (o *Object) refreshFromSource(force bool) error { + o.refreshMutex.Lock() + defer o.refreshMutex.Unlock() + var err error + var liveObject fs.Object + + if o.Object != nil && !force { + return nil + } + if o.isTempFile() { + liveObject, err = o.ParentFs.NewObject(o.Remote()) + err = errors.Wrapf(err, "in parent fs %v", o.ParentFs) + } else { + liveObject, err = o.CacheFs.Fs.NewObject(o.Remote()) + err = errors.Wrapf(err, "in cache fs %v", o.CacheFs.Fs) + } + if err != nil { + fs.Errorf(o, "error refreshing object in : %v", err) + return err + } + o.updateData(liveObject) + o.persist() + + return nil +} + +// SetModTime sets the ModTime of this object +func (o *Object) SetModTime(t time.Time) error { + if err := o.refreshFromSource(false); err != nil { + return err + } + + err := o.Object.SetModTime(t) + if err != nil { + return err + } + + o.CacheModTime = t.UnixNano() + o.persist() + fs.Debugf(o, "updated ModTime: %v", t) + + return nil +} + +// Open is used to request a specific part of the file using fs.RangeOption +func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) { + if err := o.refreshFromSource(true); err != nil { + return nil, err + } + + var err error + cacheReader := NewObjectHandle(o, o.CacheFs) + var offset, limit int64 = 0, -1 + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + case *fs.RangeOption: + offset, limit = x.Decode(o.Size()) + } + _, err = cacheReader.Seek(offset, io.SeekStart) + if err != nil { + return nil, err + } + } + + return readers.NewLimitedReadCloser(cacheReader, limit), nil +} + +// Update will change the object data +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + if err := o.refreshFromSource(false); err != nil { + return err + } + // pause background uploads if active + if o.CacheFs.opt.TempWritePath != "" { + o.CacheFs.backgroundRunner.pause() + defer o.CacheFs.backgroundRunner.play() + // don't allow started uploads + if o.isTempFile() && o.tempFileStartedUpload() { + return errors.Errorf("%v is currently uploading, can't update", o) + } + } + fs.Debugf(o, "updating object contents with size %v", src.Size()) + + // FIXME use reliable upload + err := o.Object.Update(in, src, options...) + if err != nil { + fs.Errorf(o, "error updating source: %v", err) + return err + } + + // deleting cached chunks and info to be replaced with new ones + _ = o.CacheFs.cache.RemoveObject(o.abs()) + // advertise to ChangeNotify if wrapped doesn't do that + o.CacheFs.notifyChangeUpstreamIfNeeded(o.Remote(), fs.EntryObject) + + o.CacheModTime = src.ModTime().UnixNano() + o.CacheSize = src.Size() + o.CacheHashes = make(map[hash.Type]string) + o.CacheTs = time.Now() + o.persist() + + return nil +} + +// Remove deletes the object from both the cache and the source +func (o *Object) Remove() error { + if err := o.refreshFromSource(false); err != nil { + return err + } + // pause background uploads if active + if o.CacheFs.opt.TempWritePath != "" { + o.CacheFs.backgroundRunner.pause() + defer o.CacheFs.backgroundRunner.play() + // don't allow started uploads + if o.isTempFile() && o.tempFileStartedUpload() { + return errors.Errorf("%v is currently uploading, can't delete", o) + } + } + err := o.Object.Remove() + if err != nil { + return err + } + + fs.Debugf(o, "removing object") + _ = o.CacheFs.cache.RemoveObject(o.abs()) + _ = o.CacheFs.cache.removePendingUpload(o.abs()) + parentCd := NewDirectory(o.CacheFs, cleanPath(path.Dir(o.Remote()))) + _ = o.CacheFs.cache.ExpireDir(parentCd) + // advertise to ChangeNotify if wrapped doesn't do that + o.CacheFs.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory) + + return nil +} + +// Hash requests a hash of the object and stores in the cache +// since it might or might not be called, this is lazy loaded +func (o *Object) Hash(ht hash.Type) (string, error) { + _ = o.refresh() + if o.CacheHashes == nil { + o.CacheHashes = make(map[hash.Type]string) + } + + cachedHash, found := o.CacheHashes[ht] + if found { + return cachedHash, nil + } + if err := o.refreshFromSource(false); err != nil { + return "", err + } + liveHash, err := o.Object.Hash(ht) + if err != nil { + return "", err + } + o.CacheHashes[ht] = liveHash + + o.persist() + fs.Debugf(o, "object hash cached: %v", liveHash) + + return liveHash, nil +} + +// persist adds this object to the persistent cache +func (o *Object) persist() *Object { + err := o.CacheFs.cache.AddObject(o) + if err != nil { + fs.Errorf(o, "failed to cache object: %v", err) + } + return o +} + +func (o *Object) isTempFile() bool { + _, err := o.CacheFs.cache.SearchPendingUpload(o.abs()) + if err != nil { + o.CacheType = objectInCache + return false + } + + o.CacheType = objectPendingUpload + return true +} + +func (o *Object) tempFileStartedUpload() bool { + started, err := o.CacheFs.cache.SearchPendingUpload(o.abs()) + if err != nil { + return false + } + return started +} + +// UnWrap returns the Object that this Object is wrapping or +// nil if it isn't wrapping anything +func (o *Object) UnWrap() fs.Object { + return o.Object +} + +var ( + _ fs.Object = (*Object)(nil) + _ fs.ObjectUnWrapper = (*Object)(nil) +) diff --git a/.rclone_repo/backend/cache/plex.go b/.rclone_repo/backend/cache/plex.go new file mode 100755 index 0000000..bac387d --- /dev/null +++ b/.rclone_repo/backend/cache/plex.go @@ -0,0 +1,284 @@ +// +build !plan9 + +package cache + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "sync" + + "bytes" + "io/ioutil" + + "github.com/ncw/rclone/fs" + "github.com/patrickmn/go-cache" + "golang.org/x/net/websocket" +) + +const ( + // defPlexLoginURL is the default URL for Plex login + defPlexLoginURL = "https://plex.tv/users/sign_in.json" + defPlexNotificationURL = "%s/:/websockets/notifications?X-Plex-Token=%s" +) + +// PlaySessionStateNotification is part of the API response of Plex +type PlaySessionStateNotification struct { + SessionKey string `json:"sessionKey"` + GUID string `json:"guid"` + Key string `json:"key"` + ViewOffset int64 `json:"viewOffset"` + State string `json:"state"` + TranscodeSession string `json:"transcodeSession"` +} + +// NotificationContainer is part of the API response of Plex +type NotificationContainer struct { + Type string `json:"type"` + Size int `json:"size"` + PlaySessionState []PlaySessionStateNotification `json:"PlaySessionStateNotification"` +} + +// PlexNotification is part of the API response of Plex +type PlexNotification struct { + Container NotificationContainer `json:"NotificationContainer"` +} + +// plexConnector is managing the cache integration with Plex +type plexConnector struct { + url *url.URL + username string + password string + token string + f *Fs + mu sync.Mutex + running bool + runningMu sync.Mutex + stateCache *cache.Cache + saveToken func(string) +} + +// newPlexConnector connects to a Plex server and generates a token +func newPlexConnector(f *Fs, plexURL, username, password string, saveToken func(string)) (*plexConnector, error) { + u, err := url.ParseRequestURI(strings.TrimRight(plexURL, "/")) + if err != nil { + return nil, err + } + + pc := &plexConnector{ + f: f, + url: u, + username: username, + password: password, + token: "", + stateCache: cache.New(time.Hour, time.Minute), + saveToken: saveToken, + } + + return pc, nil +} + +// newPlexConnector connects to a Plex server and generates a token +func newPlexConnectorWithToken(f *Fs, plexURL, token string) (*plexConnector, error) { + u, err := url.ParseRequestURI(strings.TrimRight(plexURL, "/")) + if err != nil { + return nil, err + } + + pc := &plexConnector{ + f: f, + url: u, + token: token, + stateCache: cache.New(time.Hour, time.Minute), + } + pc.listenWebsocket() + + return pc, nil +} + +func (p *plexConnector) closeWebsocket() { + p.runningMu.Lock() + defer p.runningMu.Unlock() + fs.Infof("plex", "stopped Plex watcher") + p.running = false +} + +func (p *plexConnector) listenWebsocket() { + p.runningMu.Lock() + defer p.runningMu.Unlock() + + u := strings.Replace(p.url.String(), "http://", "ws://", 1) + u = strings.Replace(u, "https://", "wss://", 1) + conn, err := websocket.Dial(fmt.Sprintf(defPlexNotificationURL, strings.TrimRight(u, "/"), p.token), + "", "http://localhost") + if err != nil { + fs.Errorf("plex", "%v", err) + return + } + + p.running = true + go func() { + for { + if !p.isConnected() { + break + } + + notif := &PlexNotification{} + err := websocket.JSON.Receive(conn, notif) + if err != nil { + fs.Debugf("plex", "%v", err) + p.closeWebsocket() + break + } + // we're only interested in play events + if notif.Container.Type == "playing" { + // we loop through each of them + for _, v := range notif.Container.PlaySessionState { + // event type of playing + if v.State == "playing" { + // if it's not cached get the details and cache them + if _, found := p.stateCache.Get(v.Key); !found { + req, err := http.NewRequest("GET", fmt.Sprintf("%s%s", p.url.String(), v.Key), nil) + if err != nil { + continue + } + p.fillDefaultHeaders(req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + continue + } + var data []byte + data, err = ioutil.ReadAll(resp.Body) + if err != nil { + continue + } + p.stateCache.Set(v.Key, data, cache.DefaultExpiration) + } + } else if v.State == "stopped" { + p.stateCache.Delete(v.Key) + } + } + } + } + }() +} + +// fillDefaultHeaders will add common headers to requests +func (p *plexConnector) fillDefaultHeaders(req *http.Request) { + req.Header.Add("X-Plex-Client-Identifier", fmt.Sprintf("rclone (%v)", p.f.String())) + req.Header.Add("X-Plex-Product", fmt.Sprintf("rclone (%v)", p.f.Name())) + req.Header.Add("X-Plex-Version", fs.Version) + req.Header.Add("Accept", "application/json") + if p.token != "" { + req.Header.Add("X-Plex-Token", p.token) + } +} + +// authenticate will generate a token based on a username/password +func (p *plexConnector) authenticate() error { + p.mu.Lock() + defer p.mu.Unlock() + + form := url.Values{} + form.Set("user[login]", p.username) + form.Add("user[password]", p.password) + req, err := http.NewRequest("POST", defPlexLoginURL, strings.NewReader(form.Encode())) + if err != nil { + return err + } + p.fillDefaultHeaders(req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + var data map[string]interface{} + err = json.NewDecoder(resp.Body).Decode(&data) + if err != nil { + return fmt.Errorf("failed to obtain token: %v", err) + } + tokenGen, ok := get(data, "user", "authToken") + if !ok { + return fmt.Errorf("failed to obtain token: %v", data) + } + token, ok := tokenGen.(string) + if !ok { + return fmt.Errorf("failed to obtain token: %v", data) + } + p.token = token + if p.token != "" { + if p.saveToken != nil { + p.saveToken(p.token) + } + fs.Infof(p.f.Name(), "Connected to Plex server: %v", p.url.String()) + } + p.listenWebsocket() + + return nil +} + +// isConnected checks if this rclone is authenticated to Plex +func (p *plexConnector) isConnected() bool { + p.runningMu.Lock() + defer p.runningMu.Unlock() + return p.running +} + +// isConfigured checks if this rclone is configured to use a Plex server +func (p *plexConnector) isConfigured() bool { + return p.url != nil +} + +func (p *plexConnector) isPlaying(co *Object) bool { + var err error + if !p.isConnected() { + p.listenWebsocket() + } + + remote := co.Remote() + if cr, yes := p.f.isWrappedByCrypt(); yes { + remote, err = cr.DecryptFileName(co.Remote()) + if err != nil { + fs.Debugf("plex", "can not decrypt wrapped file: %v", err) + return false + } + } + + isPlaying := false + for _, v := range p.stateCache.Items() { + if bytes.Contains(v.Object.([]byte), []byte(remote)) { + isPlaying = true + break + } + } + + return isPlaying +} + +// adapted from: https://stackoverflow.com/a/28878037 (credit) +func get(m interface{}, path ...interface{}) (interface{}, bool) { + for _, p := range path { + switch idx := p.(type) { + case string: + if mm, ok := m.(map[string]interface{}); ok { + if val, found := mm[idx]; found { + m = val + continue + } + } + return nil, false + case int: + if mm, ok := m.([]interface{}); ok { + if len(mm) > idx { + m = mm[idx] + continue + } + } + return nil, false + } + } + return m, true +} diff --git a/.rclone_repo/backend/cache/storage_memory.go b/.rclone_repo/backend/cache/storage_memory.go new file mode 100755 index 0000000..2ea0f8c --- /dev/null +++ b/.rclone_repo/backend/cache/storage_memory.go @@ -0,0 +1,98 @@ +// +build !plan9 + +package cache + +import ( + "strconv" + "strings" + "time" + + "github.com/ncw/rclone/fs" + "github.com/patrickmn/go-cache" + "github.com/pkg/errors" +) + +// Memory is a wrapper of transient storage for a go-cache store +type Memory struct { + db *cache.Cache +} + +// NewMemory builds this cache storage +// defaultExpiration will set the expiry time of chunks in this storage +func NewMemory(defaultExpiration time.Duration) *Memory { + mem := &Memory{} + err := mem.Connect(defaultExpiration) + if err != nil { + fs.Errorf("cache", "can't open ram connection: %v", err) + } + + return mem +} + +// Connect will create a connection for the storage +func (m *Memory) Connect(defaultExpiration time.Duration) error { + m.db = cache.New(defaultExpiration, -1) + return nil +} + +// HasChunk confirms the existence of a single chunk of an object +func (m *Memory) HasChunk(cachedObject *Object, offset int64) bool { + key := cachedObject.abs() + "-" + strconv.FormatInt(offset, 10) + _, found := m.db.Get(key) + return found +} + +// GetChunk will retrieve a single chunk which belongs to a cached object or an error if it doesn't find it +func (m *Memory) GetChunk(cachedObject *Object, offset int64) ([]byte, error) { + key := cachedObject.abs() + "-" + strconv.FormatInt(offset, 10) + var data []byte + + if x, found := m.db.Get(key); found { + data = x.([]byte) + return data, nil + } + + return nil, errors.Errorf("couldn't get cached object data at offset %v", offset) +} + +// AddChunk adds a new chunk of a cached object +func (m *Memory) AddChunk(fp string, data []byte, offset int64) error { + return m.AddChunkAhead(fp, data, offset, time.Second) +} + +// AddChunkAhead adds a new chunk of a cached object +func (m *Memory) AddChunkAhead(fp string, data []byte, offset int64, t time.Duration) error { + key := fp + "-" + strconv.FormatInt(offset, 10) + m.db.Set(key, data, cache.DefaultExpiration) + + return nil +} + +// CleanChunksByAge will cleanup on a cron basis +func (m *Memory) CleanChunksByAge(chunkAge time.Duration) { + m.db.DeleteExpired() +} + +// CleanChunksByNeed will cleanup chunks after the FS passes a specific chunk +func (m *Memory) CleanChunksByNeed(offset int64) { + var items map[string]cache.Item + + items = m.db.Items() + for key := range items { + sepIdx := strings.LastIndex(key, "-") + keyOffset, err := strconv.ParseInt(key[sepIdx+1:], 10, 64) + if err != nil { + fs.Errorf("cache", "couldn't parse offset entry %v", key) + continue + } + + if keyOffset < offset { + m.db.Delete(key) + } + } +} + +// CleanChunksBySize will cleanup chunks after the total size passes a certain point +func (m *Memory) CleanChunksBySize(maxSize int64) { + // NOOP +} diff --git a/.rclone_repo/backend/cache/storage_persistent.go b/.rclone_repo/backend/cache/storage_persistent.go new file mode 100755 index 0000000..57406bb --- /dev/null +++ b/.rclone_repo/backend/cache/storage_persistent.go @@ -0,0 +1,1100 @@ +// +build !plan9 + +package cache + +import ( + "time" + + "bytes" + "encoding/binary" + "encoding/json" + "os" + "path" + "strconv" + "strings" + "sync" + + "io/ioutil" + + "fmt" + + bolt "github.com/coreos/bbolt" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/walk" + "github.com/pkg/errors" +) + +// Constants +const ( + RootBucket = "root" + RootTsBucket = "rootTs" + DataTsBucket = "dataTs" + tempBucket = "pending" +) + +// Features flags for this storage type +type Features struct { + PurgeDb bool // purge the db before starting + DbWaitTime time.Duration // time to wait for DB to be available +} + +var boltMap = make(map[string]*Persistent) +var boltMapMx sync.Mutex + +// GetPersistent returns a single instance for the specific store +func GetPersistent(dbPath, chunkPath string, f *Features) (*Persistent, error) { + // write lock to create one + boltMapMx.Lock() + defer boltMapMx.Unlock() + if b, ok := boltMap[dbPath]; ok { + if !b.open { + err := b.connect() + if err != nil { + return nil, err + } + } + return b, nil + } + + bb, err := newPersistent(dbPath, chunkPath, f) + if err != nil { + return nil, err + } + boltMap[dbPath] = bb + return boltMap[dbPath], nil +} + +type chunkInfo struct { + Path string + Offset int64 + Size int64 +} + +type tempUploadInfo struct { + DestPath string + AddedOn time.Time + Started bool +} + +// String representation of a tempUploadInfo +func (t *tempUploadInfo) String() string { + return fmt.Sprintf("%v - %v (%v)", t.DestPath, t.Started, t.AddedOn) +} + +// Persistent is a wrapper of persistent storage for a bolt.DB file +type Persistent struct { + dbPath string + dataPath string + open bool + db *bolt.DB + cleanupMux sync.Mutex + tempQueueMux sync.Mutex + features *Features +} + +// newPersistent builds a new wrapper and connects to the bolt.DB file +func newPersistent(dbPath, chunkPath string, f *Features) (*Persistent, error) { + b := &Persistent{ + dbPath: dbPath, + dataPath: chunkPath, + features: f, + } + + err := b.connect() + if err != nil { + fs.Errorf(dbPath, "Error opening storage cache. Is there another rclone running on the same remote? %v", err) + return nil, err + } + + return b, nil +} + +// String will return a human friendly string for this DB (currently the dbPath) +func (b *Persistent) String() string { + return " " + b.dbPath +} + +// connect creates a connection to the configured file +// refreshDb will delete the file before to create an empty DB if it's set to true +func (b *Persistent) connect() error { + var err error + + err = os.MkdirAll(b.dataPath, os.ModePerm) + if err != nil { + return errors.Wrapf(err, "failed to create a data directory %q", b.dataPath) + } + b.db, err = bolt.Open(b.dbPath, 0644, &bolt.Options{Timeout: b.features.DbWaitTime}) + if err != nil { + return errors.Wrapf(err, "failed to open a cache connection to %q", b.dbPath) + } + if b.features.PurgeDb { + b.Purge() + } + _ = b.db.Update(func(tx *bolt.Tx) error { + _, _ = tx.CreateBucketIfNotExists([]byte(RootBucket)) + _, _ = tx.CreateBucketIfNotExists([]byte(RootTsBucket)) + _, _ = tx.CreateBucketIfNotExists([]byte(DataTsBucket)) + _, _ = tx.CreateBucketIfNotExists([]byte(tempBucket)) + + return nil + }) + + b.open = true + return nil +} + +// getBucket prepares and cleans a specific path of the form: /var/tmp and will iterate through each path component +// to get to the nested bucket of the final part (in this example: tmp) +func (b *Persistent) getBucket(dir string, createIfMissing bool, tx *bolt.Tx) *bolt.Bucket { + cleanPath(dir) + + entries := strings.FieldsFunc(dir, func(c rune) bool { + // cover Windows where rclone still uses '/' as path separator + // this should be safe as '/' is not a valid Windows character + return (os.PathSeparator == c || c == rune('/')) + }) + bucket := tx.Bucket([]byte(RootBucket)) + + for _, entry := range entries { + if createIfMissing { + bucket, _ = bucket.CreateBucketIfNotExists([]byte(entry)) + } else { + bucket = bucket.Bucket([]byte(entry)) + } + + if bucket == nil { + return nil + } + } + + return bucket +} + +// GetDir will retrieve data of a cached directory +func (b *Persistent) GetDir(remote string) (*Directory, error) { + cd := &Directory{} + + err := b.db.View(func(tx *bolt.Tx) error { + bucket := b.getBucket(remote, false, tx) + if bucket == nil { + return errors.Errorf("couldn't open bucket (%v)", remote) + } + + data := bucket.Get([]byte(".")) + if data != nil { + return json.Unmarshal(data, cd) + } + + return errors.Errorf("%v not found", remote) + }) + + return cd, err +} + +// AddDir will update a CachedDirectory metadata and all its entries +func (b *Persistent) AddDir(cachedDir *Directory) error { + return b.AddBatchDir([]*Directory{cachedDir}) +} + +// AddBatchDir will update a list of CachedDirectory metadata and all their entries +func (b *Persistent) AddBatchDir(cachedDirs []*Directory) error { + if len(cachedDirs) == 0 { + return nil + } + + return b.db.Update(func(tx *bolt.Tx) error { + var bucket *bolt.Bucket + if cachedDirs[0].Dir == "" { + bucket = tx.Bucket([]byte(RootBucket)) + } else { + bucket = b.getBucket(cachedDirs[0].Dir, true, tx) + } + if bucket == nil { + return errors.Errorf("couldn't open bucket (%v)", cachedDirs[0].Dir) + } + + for _, cachedDir := range cachedDirs { + var b *bolt.Bucket + var err error + if cachedDir.Name == "" { + b = bucket + } else { + b, err = bucket.CreateBucketIfNotExists([]byte(cachedDir.Name)) + } + if err != nil { + return err + } + + encoded, err := json.Marshal(cachedDir) + if err != nil { + return errors.Errorf("couldn't marshal object (%v): %v", cachedDir, err) + } + err = b.Put([]byte("."), encoded) + if err != nil { + return err + } + } + return nil + }) +} + +// GetDirEntries will return a CachedDirectory, its list of dir entries and/or an error if it encountered issues +func (b *Persistent) GetDirEntries(cachedDir *Directory) (fs.DirEntries, error) { + var dirEntries fs.DirEntries + + err := b.db.View(func(tx *bolt.Tx) error { + bucket := b.getBucket(cachedDir.abs(), false, tx) + if bucket == nil { + return errors.Errorf("couldn't open bucket (%v)", cachedDir.abs()) + } + + val := bucket.Get([]byte(".")) + if val != nil { + err := json.Unmarshal(val, cachedDir) + if err != nil { + return errors.Errorf("error during unmarshalling obj: %v", err) + } + } else { + return errors.Errorf("missing cached dir: %v", cachedDir) + } + + c := bucket.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + // ignore metadata key: . + if bytes.Equal(k, []byte(".")) { + continue + } + entryPath := path.Join(cachedDir.Remote(), string(k)) + + if v == nil { // directory + // we try to find a cached meta for the dir + currentBucket := c.Bucket().Bucket(k) + if currentBucket == nil { + return errors.Errorf("couldn't open bucket (%v)", string(k)) + } + + metaKey := currentBucket.Get([]byte(".")) + d := NewDirectory(cachedDir.CacheFs, entryPath) + if metaKey != nil { //if we don't find it, we create an empty dir + err := json.Unmarshal(metaKey, d) + if err != nil { // if even this fails, we fallback to an empty dir + fs.Debugf(string(k), "error during unmarshalling obj: %v", err) + } + } + + dirEntries = append(dirEntries, d) + } else { // object + o := NewObject(cachedDir.CacheFs, entryPath) + err := json.Unmarshal(v, o) + if err != nil { + fs.Debugf(string(k), "error during unmarshalling obj: %v", err) + continue + } + + dirEntries = append(dirEntries, o) + } + } + + return nil + }) + + return dirEntries, err +} + +// RemoveDir will delete a CachedDirectory, all its objects and all the chunks stored for it +func (b *Persistent) RemoveDir(fp string) error { + var err error + parentDir, dirName := path.Split(fp) + if fp == "" { + err = b.db.Update(func(tx *bolt.Tx) error { + err := tx.DeleteBucket([]byte(RootBucket)) + if err != nil { + fs.Debugf(fp, "couldn't delete from cache: %v", err) + return err + } + _, _ = tx.CreateBucketIfNotExists([]byte(RootBucket)) + return nil + }) + } else { + err = b.db.Update(func(tx *bolt.Tx) error { + bucket := b.getBucket(cleanPath(parentDir), false, tx) + if bucket == nil { + return errors.Errorf("couldn't open bucket (%v)", fp) + } + // delete the cached dir + err := bucket.DeleteBucket([]byte(cleanPath(dirName))) + if err != nil { + fs.Debugf(fp, "couldn't delete from cache: %v", err) + } + return nil + }) + } + + // delete chunks on disk + // safe to ignore as the files might not have been open + if err == nil { + _ = os.RemoveAll(path.Join(b.dataPath, fp)) + _ = os.MkdirAll(b.dataPath, os.ModePerm) + } + + return err +} + +// ExpireDir will flush a CachedDirectory and all its objects from the objects +// chunks will remain as they are +func (b *Persistent) ExpireDir(cd *Directory) error { + t := time.Now().Add(time.Duration(-cd.CacheFs.opt.InfoAge)) + cd.CacheTs = &t + + // expire all parents + return b.db.Update(func(tx *bolt.Tx) error { + // expire all the parents + currentDir := cd.abs() + for { // until we get to the root + bucket := b.getBucket(currentDir, false, tx) + if bucket != nil { + val := bucket.Get([]byte(".")) + if val != nil { + cd2 := &Directory{CacheFs: cd.CacheFs} + err := json.Unmarshal(val, cd2) + if err == nil { + fs.Debugf(cd, "cache: expired %v", currentDir) + cd2.CacheTs = &t + enc2, _ := json.Marshal(cd2) + _ = bucket.Put([]byte("."), enc2) + } + } + } + if currentDir == "" { + break + } + currentDir = cleanPath(path.Dir(currentDir)) + } + return nil + }) +} + +// GetObject will return a CachedObject from its parent directory or an error if it doesn't find it +func (b *Persistent) GetObject(cachedObject *Object) (err error) { + return b.db.View(func(tx *bolt.Tx) error { + bucket := b.getBucket(cachedObject.Dir, false, tx) + if bucket == nil { + return errors.Errorf("couldn't open parent bucket for %v", cachedObject.Dir) + } + val := bucket.Get([]byte(cachedObject.Name)) + if val != nil { + return json.Unmarshal(val, cachedObject) + } + return errors.Errorf("couldn't find object (%v)", cachedObject.Name) + }) +} + +// AddObject will create a cached object in its parent directory +func (b *Persistent) AddObject(cachedObject *Object) error { + return b.db.Update(func(tx *bolt.Tx) error { + bucket := b.getBucket(cachedObject.Dir, true, tx) + if bucket == nil { + return errors.Errorf("couldn't open parent bucket for %v", cachedObject) + } + // cache Object Info + encoded, err := json.Marshal(cachedObject) + if err != nil { + return errors.Errorf("couldn't marshal object (%v) info: %v", cachedObject, err) + } + err = bucket.Put([]byte(cachedObject.Name), []byte(encoded)) + if err != nil { + return errors.Errorf("couldn't cache object (%v) info: %v", cachedObject, err) + } + return nil + }) +} + +// RemoveObject will delete a single cached object and all the chunks which belong to it +func (b *Persistent) RemoveObject(fp string) error { + parentDir, objName := path.Split(fp) + return b.db.Update(func(tx *bolt.Tx) error { + bucket := b.getBucket(cleanPath(parentDir), false, tx) + if bucket == nil { + return errors.Errorf("couldn't open parent bucket for %v", cleanPath(parentDir)) + } + err := bucket.Delete([]byte(cleanPath(objName))) + if err != nil { + fs.Debugf(fp, "couldn't delete obj from storage: %v", err) + } + // delete chunks on disk + // safe to ignore as the file might not have been open + _ = os.RemoveAll(path.Join(b.dataPath, fp)) + return nil + }) +} + +// ExpireObject will flush an Object and all its data if desired +func (b *Persistent) ExpireObject(co *Object, withData bool) error { + co.CacheTs = time.Now().Add(time.Duration(-co.CacheFs.opt.InfoAge)) + err := b.AddObject(co) + if withData { + _ = os.RemoveAll(path.Join(b.dataPath, co.abs())) + } + return err +} + +// HasEntry confirms the existence of a single entry (dir or object) +func (b *Persistent) HasEntry(remote string) bool { + dir, name := path.Split(remote) + dir = cleanPath(dir) + name = cleanPath(name) + + err := b.db.View(func(tx *bolt.Tx) error { + bucket := b.getBucket(dir, false, tx) + if bucket == nil { + return errors.Errorf("couldn't open parent bucket for %v", remote) + } + if f := bucket.Bucket([]byte(name)); f != nil { + return nil + } + if f := bucket.Get([]byte(name)); f != nil { + return nil + } + + return errors.Errorf("couldn't find object (%v)", remote) + }) + if err == nil { + return true + } + return false +} + +// HasChunk confirms the existence of a single chunk of an object +func (b *Persistent) HasChunk(cachedObject *Object, offset int64) bool { + fp := path.Join(b.dataPath, cachedObject.abs(), strconv.FormatInt(offset, 10)) + if _, err := os.Stat(fp); !os.IsNotExist(err) { + return true + } + return false +} + +// GetChunk will retrieve a single chunk which belongs to a cached object or an error if it doesn't find it +func (b *Persistent) GetChunk(cachedObject *Object, offset int64) ([]byte, error) { + var data []byte + + fp := path.Join(b.dataPath, cachedObject.abs(), strconv.FormatInt(offset, 10)) + data, err := ioutil.ReadFile(fp) + if err != nil { + return nil, err + } + + return data, err +} + +// AddChunk adds a new chunk of a cached object +func (b *Persistent) AddChunk(fp string, data []byte, offset int64) error { + _ = os.MkdirAll(path.Join(b.dataPath, fp), os.ModePerm) + + filePath := path.Join(b.dataPath, fp, strconv.FormatInt(offset, 10)) + err := ioutil.WriteFile(filePath, data, os.ModePerm) + if err != nil { + return err + } + + return b.db.Update(func(tx *bolt.Tx) error { + tsBucket := tx.Bucket([]byte(DataTsBucket)) + ts := time.Now() + found := false + + // delete (older) timestamps for the same object + c := tsBucket.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + var ci chunkInfo + err = json.Unmarshal(v, &ci) + if err != nil { + continue + } + if ci.Path == fp && ci.Offset == offset { + if tsInCache := time.Unix(0, btoi(k)); tsInCache.After(ts) && !found { + found = true + continue + } + err := c.Delete() + if err != nil { + fs.Debugf(fp, "failed to clean chunk: %v", err) + } + } + } + // don't overwrite if a newer one is already there + if found { + return nil + } + enc, err := json.Marshal(chunkInfo{Path: fp, Offset: offset, Size: int64(len(data))}) + if err != nil { + fs.Debugf(fp, "failed to timestamp chunk: %v", err) + } + err = tsBucket.Put(itob(ts.UnixNano()), enc) + if err != nil { + fs.Debugf(fp, "failed to timestamp chunk: %v", err) + } + return nil + }) +} + +// CleanChunksByAge will cleanup on a cron basis +func (b *Persistent) CleanChunksByAge(chunkAge time.Duration) { + // NOOP +} + +// CleanChunksByNeed is a noop for this implementation +func (b *Persistent) CleanChunksByNeed(offset int64) { + // noop: we want to clean a Bolt DB by time only +} + +// CleanChunksBySize will cleanup chunks after the total size passes a certain point +func (b *Persistent) CleanChunksBySize(maxSize int64) { + b.cleanupMux.Lock() + defer b.cleanupMux.Unlock() + var cntChunks int + var roughlyCleaned fs.SizeSuffix + + err := b.db.Update(func(tx *bolt.Tx) error { + dataTsBucket := tx.Bucket([]byte(DataTsBucket)) + if dataTsBucket == nil { + return errors.Errorf("Couldn't open (%v) bucket", DataTsBucket) + } + // iterate through ts + c := dataTsBucket.Cursor() + totalSize := int64(0) + for k, v := c.First(); k != nil; k, v = c.Next() { + var ci chunkInfo + err := json.Unmarshal(v, &ci) + if err != nil { + continue + } + + totalSize += ci.Size + } + + if totalSize > maxSize { + needToClean := totalSize - maxSize + roughlyCleaned = fs.SizeSuffix(needToClean) + for k, v := c.First(); k != nil; k, v = c.Next() { + var ci chunkInfo + err := json.Unmarshal(v, &ci) + if err != nil { + continue + } + // delete this ts entry + err = c.Delete() + if err != nil { + fs.Errorf(ci.Path, "failed deleting chunk ts during cleanup (%v): %v", ci.Offset, err) + continue + } + err = os.Remove(path.Join(b.dataPath, ci.Path, strconv.FormatInt(ci.Offset, 10))) + if err == nil { + cntChunks++ + needToClean -= ci.Size + if needToClean <= 0 { + break + } + } + } + } + if cntChunks > 0 { + fs.Infof("cache-cleanup", "chunks %v, est. size: %v", cntChunks, roughlyCleaned.String()) + + } + return nil + }) + + if err != nil { + if err == bolt.ErrDatabaseNotOpen { + // we're likely a late janitor and we need to end quietly as there's no guarantee of what exists anymore + return + } + fs.Errorf("cache", "cleanup failed: %v", err) + } +} + +// Stats returns a go map with the stats key values +func (b *Persistent) Stats() (map[string]map[string]interface{}, error) { + r := make(map[string]map[string]interface{}) + r["data"] = make(map[string]interface{}) + r["data"]["oldest-ts"] = time.Now() + r["data"]["oldest-file"] = "" + r["data"]["newest-ts"] = time.Now() + r["data"]["newest-file"] = "" + r["data"]["total-chunks"] = 0 + r["data"]["total-size"] = int64(0) + r["files"] = make(map[string]interface{}) + r["files"]["oldest-ts"] = time.Now() + r["files"]["oldest-name"] = "" + r["files"]["newest-ts"] = time.Now() + r["files"]["newest-name"] = "" + r["files"]["total-files"] = 0 + + _ = b.db.View(func(tx *bolt.Tx) error { + dataTsBucket := tx.Bucket([]byte(DataTsBucket)) + rootTsBucket := tx.Bucket([]byte(RootTsBucket)) + + var totalDirs int + var totalFiles int + _ = b.iterateBuckets(tx.Bucket([]byte(RootBucket)), func(name string) { + totalDirs++ + }, func(key string, val []byte) { + totalFiles++ + }) + r["files"]["total-dir"] = totalDirs + r["files"]["total-files"] = totalFiles + + c := dataTsBucket.Cursor() + + totalChunks := 0 + totalSize := int64(0) + for k, v := c.First(); k != nil; k, v = c.Next() { + var ci chunkInfo + err := json.Unmarshal(v, &ci) + if err != nil { + continue + } + totalChunks++ + totalSize += ci.Size + } + r["data"]["total-chunks"] = totalChunks + r["data"]["total-size"] = totalSize + + if k, v := c.First(); k != nil { + var ci chunkInfo + _ = json.Unmarshal(v, &ci) + r["data"]["oldest-ts"] = time.Unix(0, btoi(k)) + r["data"]["oldest-file"] = ci.Path + } + if k, v := c.Last(); k != nil { + var ci chunkInfo + _ = json.Unmarshal(v, &ci) + r["data"]["newest-ts"] = time.Unix(0, btoi(k)) + r["data"]["newest-file"] = ci.Path + } + + c = rootTsBucket.Cursor() + if k, v := c.First(); k != nil { + // split to get (abs path - offset) + r["files"]["oldest-ts"] = time.Unix(0, btoi(k)) + r["files"]["oldest-name"] = string(v) + } + if k, v := c.Last(); k != nil { + r["files"]["newest-ts"] = time.Unix(0, btoi(k)) + r["files"]["newest-name"] = string(v) + } + + return nil + }) + + return r, nil +} + +// Purge will flush the entire cache +func (b *Persistent) Purge() { + b.cleanupMux.Lock() + defer b.cleanupMux.Unlock() + + _ = b.db.Update(func(tx *bolt.Tx) error { + _ = tx.DeleteBucket([]byte(RootBucket)) + _ = tx.DeleteBucket([]byte(RootTsBucket)) + _ = tx.DeleteBucket([]byte(DataTsBucket)) + + _, _ = tx.CreateBucketIfNotExists([]byte(RootBucket)) + _, _ = tx.CreateBucketIfNotExists([]byte(RootTsBucket)) + _, _ = tx.CreateBucketIfNotExists([]byte(DataTsBucket)) + + return nil + }) + + err := os.RemoveAll(b.dataPath) + if err != nil { + fs.Errorf(b, "issue removing data folder: %v", err) + } + err = os.MkdirAll(b.dataPath, os.ModePerm) + if err != nil { + fs.Errorf(b, "issue removing data folder: %v", err) + } +} + +// GetChunkTs retrieves the current timestamp of this chunk +func (b *Persistent) GetChunkTs(path string, offset int64) (time.Time, error) { + var t time.Time + + err := b.db.View(func(tx *bolt.Tx) error { + tsBucket := tx.Bucket([]byte(DataTsBucket)) + c := tsBucket.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + var ci chunkInfo + err := json.Unmarshal(v, &ci) + if err != nil { + continue + } + if ci.Path == path && ci.Offset == offset { + t = time.Unix(0, btoi(k)) + return nil + } + } + return errors.Errorf("not found %v-%v", path, offset) + }) + + return t, err +} + +func (b *Persistent) iterateBuckets(buk *bolt.Bucket, bucketFn func(name string), kvFn func(key string, val []byte)) error { + err := b.db.View(func(tx *bolt.Tx) error { + var c *bolt.Cursor + if buk == nil { + c = tx.Cursor() + } else { + c = buk.Cursor() + } + for k, v := c.First(); k != nil; k, v = c.Next() { + if v == nil { + var buk2 *bolt.Bucket + if buk == nil { + buk2 = tx.Bucket(k) + } else { + buk2 = buk.Bucket(k) + } + + bucketFn(string(k)) + _ = b.iterateBuckets(buk2, bucketFn, kvFn) + } else { + kvFn(string(k), v) + } + } + return nil + }) + + return err +} + +func (b *Persistent) dumpRoot() string { + var itBuckets func(buk *bolt.Bucket) map[string]interface{} + + itBuckets = func(buk *bolt.Bucket) map[string]interface{} { + m := make(map[string]interface{}) + c := buk.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + if v == nil { + buk2 := buk.Bucket(k) + m[string(k)] = itBuckets(buk2) + } else { + m[string(k)] = "-" + } + } + return m + } + var mm map[string]interface{} + _ = b.db.View(func(tx *bolt.Tx) error { + mm = itBuckets(tx.Bucket([]byte(RootBucket))) + return nil + }) + raw, _ := json.MarshalIndent(mm, "", " ") + return string(raw) +} + +// addPendingUpload adds a new file to the pending queue of uploads +func (b *Persistent) addPendingUpload(destPath string, started bool) error { + return b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + tempObj := &tempUploadInfo{ + DestPath: destPath, + AddedOn: time.Now(), + Started: started, + } + + // cache Object Info + encoded, err := json.Marshal(tempObj) + if err != nil { + return errors.Errorf("couldn't marshal object (%v) info: %v", destPath, err) + } + err = bucket.Put([]byte(destPath), []byte(encoded)) + if err != nil { + return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err) + } + + return nil + }) +} + +// getPendingUpload returns the next file from the pending queue of uploads +func (b *Persistent) getPendingUpload(inRoot string, waitTime time.Duration) (destPath string, err error) { + b.tempQueueMux.Lock() + defer b.tempQueueMux.Unlock() + + err = b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + + c := bucket.Cursor() + for k, v := c.Seek([]byte(inRoot)); k != nil && bytes.HasPrefix(k, []byte(inRoot)); k, v = c.Next() { + //for k, v := c.First(); k != nil; k, v = c.Next() { + var tempObj = &tempUploadInfo{} + err = json.Unmarshal(v, tempObj) + if err != nil { + fs.Errorf(b, "failed to read pending upload: %v", err) + continue + } + // skip over started uploads + if tempObj.Started || time.Now().Before(tempObj.AddedOn.Add(waitTime)) { + continue + } + + tempObj.Started = true + v2, err := json.Marshal(tempObj) + if err != nil { + fs.Errorf(b, "failed to update pending upload: %v", err) + continue + } + err = bucket.Put(k, v2) + if err != nil { + fs.Errorf(b, "failed to update pending upload: %v", err) + continue + } + + destPath = tempObj.DestPath + return nil + } + + return errors.Errorf("no pending upload found") + }) + + return destPath, err +} + +// SearchPendingUpload returns the file info from the pending queue of uploads +func (b *Persistent) SearchPendingUpload(remote string) (started bool, err error) { + err = b.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(tempBucket)) + if bucket == nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + + var tempObj = &tempUploadInfo{} + v := bucket.Get([]byte(remote)) + err = json.Unmarshal(v, tempObj) + if err != nil { + return errors.Errorf("pending upload (%v) not found %v", remote, err) + } + + started = tempObj.Started + return nil + }) + + return started, err +} + +// searchPendingUploadFromDir files currently pending upload from a single dir +func (b *Persistent) searchPendingUploadFromDir(dir string) (remotes []string, err error) { + err = b.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(tempBucket)) + if bucket == nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + + c := bucket.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + var tempObj = &tempUploadInfo{} + err = json.Unmarshal(v, tempObj) + if err != nil { + fs.Errorf(b, "failed to read pending upload: %v", err) + continue + } + parentDir := cleanPath(path.Dir(tempObj.DestPath)) + if dir == parentDir { + remotes = append(remotes, tempObj.DestPath) + } + } + + return nil + }) + + return remotes, err +} + +func (b *Persistent) rollbackPendingUpload(remote string) error { + b.tempQueueMux.Lock() + defer b.tempQueueMux.Unlock() + + return b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + var tempObj = &tempUploadInfo{} + v := bucket.Get([]byte(remote)) + err = json.Unmarshal(v, tempObj) + if err != nil { + return errors.Errorf("pending upload (%v) not found %v", remote, err) + } + tempObj.Started = false + v2, err := json.Marshal(tempObj) + if err != nil { + return errors.Errorf("pending upload not updated %v", err) + } + err = bucket.Put([]byte(tempObj.DestPath), v2) + if err != nil { + return errors.Errorf("pending upload not updated %v", err) + } + return nil + }) +} + +func (b *Persistent) removePendingUpload(remote string) error { + b.tempQueueMux.Lock() + defer b.tempQueueMux.Unlock() + + return b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + return bucket.Delete([]byte(remote)) + }) +} + +// updatePendingUpload allows to update an existing item in the queue while checking if it's not started in the same +// transaction. If it is started, it will not allow the update +func (b *Persistent) updatePendingUpload(remote string, fn func(item *tempUploadInfo) error) error { + b.tempQueueMux.Lock() + defer b.tempQueueMux.Unlock() + + return b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + + var tempObj = &tempUploadInfo{} + v := bucket.Get([]byte(remote)) + err = json.Unmarshal(v, tempObj) + if err != nil { + return errors.Errorf("pending upload (%v) not found %v", remote, err) + } + if tempObj.Started { + return errors.Errorf("pending upload already started %v", remote) + } + err = fn(tempObj) + if err != nil { + return err + } + if remote != tempObj.DestPath { + err := bucket.Delete([]byte(remote)) + if err != nil { + return err + } + // if this is removed then the entry can be removed too + if tempObj.DestPath == "" { + return nil + } + } + v2, err := json.Marshal(tempObj) + if err != nil { + return errors.Errorf("pending upload not updated %v", err) + } + err = bucket.Put([]byte(tempObj.DestPath), v2) + if err != nil { + return errors.Errorf("pending upload not updated %v", err) + } + + return nil + }) +} + +// SetPendingUploadToStarted is a way to mark an entry as started (even if it's not already) +// TO BE USED IN TESTING ONLY +func (b *Persistent) SetPendingUploadToStarted(remote string) error { + return b.updatePendingUpload(remote, func(item *tempUploadInfo) error { + item.Started = true + return nil + }) +} + +// ReconcileTempUploads will recursively look for all the files in the temp directory and add them to the queue +func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error { + return b.db.Update(func(tx *bolt.Tx) error { + _ = tx.DeleteBucket([]byte(tempBucket)) + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return err + } + + var queuedEntries []fs.Object + err = walk.Walk(cacheFs.tempFs, "", true, -1, func(path string, entries fs.DirEntries, err error) error { + for _, o := range entries { + if oo, ok := o.(fs.Object); ok { + queuedEntries = append(queuedEntries, oo) + } + } + return nil + }) + if err != nil { + return err + } + + fs.Debugf(cacheFs, "reconciling temporary uploads") + for _, queuedEntry := range queuedEntries { + destPath := path.Join(cacheFs.Root(), queuedEntry.Remote()) + tempObj := &tempUploadInfo{ + DestPath: destPath, + AddedOn: time.Now(), + Started: false, + } + + // cache Object Info + encoded, err := json.Marshal(tempObj) + if err != nil { + return errors.Errorf("couldn't marshal object (%v) info: %v", queuedEntry, err) + } + err = bucket.Put([]byte(destPath), []byte(encoded)) + if err != nil { + return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err) + } + fs.Debugf(cacheFs, "reconciled temporary upload: %v", destPath) + } + + return nil + }) +} + +// PurgeTempUploads will remove all the pending uploads from the queue +// TO BE USED IN TESTING ONLY +func (b *Persistent) PurgeTempUploads() { + b.tempQueueMux.Lock() + defer b.tempQueueMux.Unlock() + + _ = b.db.Update(func(tx *bolt.Tx) error { + _ = tx.DeleteBucket([]byte(tempBucket)) + _, _ = tx.CreateBucketIfNotExists([]byte(tempBucket)) + return nil + }) +} + +// Close should be called when the program ends gracefully +func (b *Persistent) Close() { + b.cleanupMux.Lock() + defer b.cleanupMux.Unlock() + + err := b.db.Close() + if err != nil { + fs.Errorf(b, "closing handle: %v", err) + } + b.open = false +} + +// itob returns an 8-byte big endian representation of v. +func itob(v int64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(v)) + return b +} + +func btoi(d []byte) int64 { + return int64(binary.BigEndian.Uint64(d)) +} diff --git a/.rclone_repo/backend/crypt/cipher.go b/.rclone_repo/backend/crypt/cipher.go new file mode 100755 index 0000000..ec21a80 --- /dev/null +++ b/.rclone_repo/backend/crypt/cipher.go @@ -0,0 +1,1087 @@ +package crypt + +import ( + "bytes" + "crypto/aes" + gocipher "crypto/cipher" + "crypto/rand" + "encoding/base32" + "fmt" + "io" + "strconv" + "strings" + "sync" + "unicode/utf8" + + "github.com/ncw/rclone/backend/crypt/pkcs7" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/accounting" + "github.com/pkg/errors" + + "golang.org/x/crypto/nacl/secretbox" + "golang.org/x/crypto/scrypt" + + "github.com/rfjakob/eme" +) + +// Constants +const ( + nameCipherBlockSize = aes.BlockSize + fileMagic = "RCLONE\x00\x00" + fileMagicSize = len(fileMagic) + fileNonceSize = 24 + fileHeaderSize = fileMagicSize + fileNonceSize + blockHeaderSize = secretbox.Overhead + blockDataSize = 64 * 1024 + blockSize = blockHeaderSize + blockDataSize + encryptedSuffix = ".bin" // when file name encryption is off we add this suffix to make sure the cloud provider doesn't process the file +) + +// Errors returned by cipher +var ( + ErrorBadDecryptUTF8 = errors.New("bad decryption - utf-8 invalid") + ErrorBadDecryptControlChar = errors.New("bad decryption - contains control chars") + ErrorNotAMultipleOfBlocksize = errors.New("not a multiple of blocksize") + ErrorTooShortAfterDecode = errors.New("too short after base32 decode") + ErrorEncryptedFileTooShort = errors.New("file is too short to be encrypted") + ErrorEncryptedFileBadHeader = errors.New("file has truncated block header") + ErrorEncryptedBadMagic = errors.New("not an encrypted file - bad magic string") + ErrorEncryptedBadBlock = errors.New("failed to authenticate decrypted block - bad password?") + ErrorBadBase32Encoding = errors.New("bad base32 filename encoding") + ErrorFileClosed = errors.New("file already closed") + ErrorNotAnEncryptedFile = errors.New("not an encrypted file - no \"" + encryptedSuffix + "\" suffix") + ErrorBadSeek = errors.New("Seek beyond end of file") + defaultSalt = []byte{0xA8, 0x0D, 0xF4, 0x3A, 0x8F, 0xBD, 0x03, 0x08, 0xA7, 0xCA, 0xB8, 0x3E, 0x58, 0x1F, 0x86, 0xB1} + obfuscQuoteRune = '!' +) + +// Global variables +var ( + fileMagicBytes = []byte(fileMagic) +) + +// ReadSeekCloser is the interface of the read handles +type ReadSeekCloser interface { + io.Reader + io.Seeker + io.Closer + fs.RangeSeeker +} + +// OpenRangeSeek opens the file handle at the offset with the limit given +type OpenRangeSeek func(offset, limit int64) (io.ReadCloser, error) + +// Cipher is used to swap out the encryption implementations +type Cipher interface { + // EncryptFileName encrypts a file path + EncryptFileName(string) string + // DecryptFileName decrypts a file path, returns error if decrypt was invalid + DecryptFileName(string) (string, error) + // EncryptDirName encrypts a directory path + EncryptDirName(string) string + // DecryptDirName decrypts a directory path, returns error if decrypt was invalid + DecryptDirName(string) (string, error) + // EncryptData + EncryptData(io.Reader) (io.Reader, error) + // DecryptData + DecryptData(io.ReadCloser) (io.ReadCloser, error) + // DecryptDataSeek decrypt at a given position + DecryptDataSeek(open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) + // EncryptedSize calculates the size of the data when encrypted + EncryptedSize(int64) int64 + // DecryptedSize calculates the size of the data when decrypted + DecryptedSize(int64) (int64, error) + // NameEncryptionMode returns the used mode for name handling + NameEncryptionMode() NameEncryptionMode +} + +// NameEncryptionMode is the type of file name encryption in use +type NameEncryptionMode int + +// NameEncryptionMode levels +const ( + NameEncryptionOff NameEncryptionMode = iota + NameEncryptionStandard + NameEncryptionObfuscated +) + +// NewNameEncryptionMode turns a string into a NameEncryptionMode +func NewNameEncryptionMode(s string) (mode NameEncryptionMode, err error) { + s = strings.ToLower(s) + switch s { + case "off": + mode = NameEncryptionOff + case "standard": + mode = NameEncryptionStandard + case "obfuscate": + mode = NameEncryptionObfuscated + default: + err = errors.Errorf("Unknown file name encryption mode %q", s) + } + return mode, err +} + +// String turns mode into a human readable string +func (mode NameEncryptionMode) String() (out string) { + switch mode { + case NameEncryptionOff: + out = "off" + case NameEncryptionStandard: + out = "standard" + case NameEncryptionObfuscated: + out = "obfuscate" + default: + out = fmt.Sprintf("Unknown mode #%d", mode) + } + return out +} + +type cipher struct { + dataKey [32]byte // Key for secretbox + nameKey [32]byte // 16,24 or 32 bytes + nameTweak [nameCipherBlockSize]byte // used to tweak the name crypto + block gocipher.Block + mode NameEncryptionMode + buffers sync.Pool // encrypt/decrypt buffers + cryptoRand io.Reader // read crypto random numbers from here + dirNameEncrypt bool +} + +// newCipher initialises the cipher. If salt is "" then it uses a built in salt val +func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool) (*cipher, error) { + c := &cipher{ + mode: mode, + cryptoRand: rand.Reader, + dirNameEncrypt: dirNameEncrypt, + } + c.buffers.New = func() interface{} { + return make([]byte, blockSize) + } + err := c.Key(password, salt) + if err != nil { + return nil, err + } + return c, nil +} + +// Key creates all the internal keys from the password passed in using +// scrypt. +// +// If salt is "" we use a fixed salt just to make attackers lives +// slighty harder than using no salt. +// +// Note that empty passsword makes all 0x00 keys which is used in the +// tests. +func (c *cipher) Key(password, salt string) (err error) { + const keySize = len(c.dataKey) + len(c.nameKey) + len(c.nameTweak) + var saltBytes = defaultSalt + if salt != "" { + saltBytes = []byte(salt) + } + var key []byte + if password == "" { + key = make([]byte, keySize) + } else { + key, err = scrypt.Key([]byte(password), saltBytes, 16384, 8, 1, keySize) + if err != nil { + return err + } + } + copy(c.dataKey[:], key) + copy(c.nameKey[:], key[len(c.dataKey):]) + copy(c.nameTweak[:], key[len(c.dataKey)+len(c.nameKey):]) + // Key the name cipher + c.block, err = aes.NewCipher(c.nameKey[:]) + return err +} + +// getBlock gets a block from the pool of size blockSize +func (c *cipher) getBlock() []byte { + return c.buffers.Get().([]byte) +} + +// putBlock returns a block to the pool of size blockSize +func (c *cipher) putBlock(buf []byte) { + if len(buf) != blockSize { + panic("bad blocksize returned to pool") + } + c.buffers.Put(buf) +} + +// check to see if the byte string is valid with no control characters +// from 0x00 to 0x1F and is a valid UTF-8 string +func checkValidString(buf []byte) error { + for i := range buf { + c := buf[i] + if c >= 0x00 && c < 0x20 || c == 0x7F { + return ErrorBadDecryptControlChar + } + } + if !utf8.Valid(buf) { + return ErrorBadDecryptUTF8 + } + return nil +} + +// encodeFileName encodes a filename using a modified version of +// standard base32 as described in RFC4648 +// +// The standard encoding is modified in two ways +// * it becomes lower case (no-one likes upper case filenames!) +// * we strip the padding character `=` +func encodeFileName(in []byte) string { + encoded := base32.HexEncoding.EncodeToString(in) + encoded = strings.TrimRight(encoded, "=") + return strings.ToLower(encoded) +} + +// decodeFileName decodes a filename as encoded by encodeFileName +func decodeFileName(in string) ([]byte, error) { + if strings.HasSuffix(in, "=") { + return nil, ErrorBadBase32Encoding + } + // First figure out how many padding characters to add + roundUpToMultipleOf8 := (len(in) + 7) &^ 7 + equals := roundUpToMultipleOf8 - len(in) + in = strings.ToUpper(in) + "========"[:equals] + return base32.HexEncoding.DecodeString(in) +} + +// encryptSegment encrypts a path segment +// +// This uses EME with AES +// +// EME (ECB-Mix-ECB) is a wide-block encryption mode presented in the +// 2003 paper "A Parallelizable Enciphering Mode" by Halevi and +// Rogaway. +// +// This makes for determinstic encryption which is what we want - the +// same filename must encrypt to the same thing. +// +// This means that +// * filenames with the same name will encrypt the same +// * filenames which start the same won't have a common prefix +func (c *cipher) encryptSegment(plaintext string) string { + if plaintext == "" { + return "" + } + paddedPlaintext := pkcs7.Pad(nameCipherBlockSize, []byte(plaintext)) + ciphertext := eme.Transform(c.block, c.nameTweak[:], paddedPlaintext, eme.DirectionEncrypt) + return encodeFileName(ciphertext) +} + +// decryptSegment decrypts a path segment +func (c *cipher) decryptSegment(ciphertext string) (string, error) { + if ciphertext == "" { + return "", nil + } + rawCiphertext, err := decodeFileName(ciphertext) + if err != nil { + return "", err + } + if len(rawCiphertext)%nameCipherBlockSize != 0 { + return "", ErrorNotAMultipleOfBlocksize + } + if len(rawCiphertext) == 0 { + // not possible if decodeFilename() working correctly + return "", ErrorTooShortAfterDecode + } + paddedPlaintext := eme.Transform(c.block, c.nameTweak[:], rawCiphertext, eme.DirectionDecrypt) + plaintext, err := pkcs7.Unpad(nameCipherBlockSize, paddedPlaintext) + if err != nil { + return "", err + } + err = checkValidString(plaintext) + if err != nil { + return "", err + } + return string(plaintext), err +} + +// Simple obfuscation routines +func (c *cipher) obfuscateSegment(plaintext string) string { + if plaintext == "" { + return "" + } + + // If the string isn't valid UTF8 then don't rotate; just + // prepend a !. + if !utf8.ValidString(plaintext) { + return "!." + plaintext + } + + // Calculate a simple rotation based on the filename and + // the nameKey + var dir int + for _, runeValue := range plaintext { + dir += int(runeValue) + } + dir = dir % 256 + + // We'll use this number to store in the result filename... + var result bytes.Buffer + _, _ = result.WriteString(strconv.Itoa(dir) + ".") + + // but we'll augment it with the nameKey for real calculation + for i := 0; i < len(c.nameKey); i++ { + dir += int(c.nameKey[i]) + } + + // Now for each character, depending on the range it is in + // we will actually rotate a different amount + for _, runeValue := range plaintext { + switch { + case runeValue == obfuscQuoteRune: + // Quote the Quote character + _, _ = result.WriteRune(obfuscQuoteRune) + _, _ = result.WriteRune(obfuscQuoteRune) + + case runeValue >= '0' && runeValue <= '9': + // Number + thisdir := (dir % 9) + 1 + newRune := '0' + (int(runeValue)-'0'+thisdir)%10 + _, _ = result.WriteRune(rune(newRune)) + + case (runeValue >= 'A' && runeValue <= 'Z') || + (runeValue >= 'a' && runeValue <= 'z'): + // ASCII letter. Try to avoid trivial A->a mappings + thisdir := dir%25 + 1 + // Calculate the offset of this character in A-Za-z + pos := int(runeValue - 'A') + if pos >= 26 { + pos -= 6 // It's lower case + } + // Rotate the character to the new location + pos = (pos + thisdir) % 52 + if pos >= 26 { + pos += 6 // and handle lower case offset again + } + _, _ = result.WriteRune(rune('A' + pos)) + + case runeValue >= 0xA0 && runeValue <= 0xFF: + // Latin 1 supplement + thisdir := (dir % 95) + 1 + newRune := 0xA0 + (int(runeValue)-0xA0+thisdir)%96 + _, _ = result.WriteRune(rune(newRune)) + + case runeValue >= 0x100: + // Some random Unicode range; we have no good rules here + thisdir := (dir % 127) + 1 + base := int(runeValue - runeValue%256) + newRune := rune(base + (int(runeValue)-base+thisdir)%256) + // If the new character isn't a valid UTF8 char + // then don't rotate it. Quote it instead + if !utf8.ValidRune(newRune) { + _, _ = result.WriteRune(obfuscQuoteRune) + _, _ = result.WriteRune(runeValue) + } else { + _, _ = result.WriteRune(newRune) + } + + default: + // Leave character untouched + _, _ = result.WriteRune(runeValue) + } + } + return result.String() +} + +func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) { + if ciphertext == "" { + return "", nil + } + pos := strings.Index(ciphertext, ".") + if pos == -1 { + return "", ErrorNotAnEncryptedFile + } // No . + num := ciphertext[:pos] + if num == "!" { + // No rotation; probably original was not valid unicode + return ciphertext[pos+1:], nil + } + dir, err := strconv.Atoi(num) + if err != nil { + return "", ErrorNotAnEncryptedFile // Not a number + } + + // add the nameKey to get the real rotate distance + for i := 0; i < len(c.nameKey); i++ { + dir += int(c.nameKey[i]) + } + + var result bytes.Buffer + + inQuote := false + for _, runeValue := range ciphertext[pos+1:] { + switch { + case inQuote: + _, _ = result.WriteRune(runeValue) + inQuote = false + + case runeValue == obfuscQuoteRune: + inQuote = true + + case runeValue >= '0' && runeValue <= '9': + // Number + thisdir := (dir % 9) + 1 + newRune := '0' + int(runeValue) - '0' - thisdir + if newRune < '0' { + newRune += 10 + } + _, _ = result.WriteRune(rune(newRune)) + + case (runeValue >= 'A' && runeValue <= 'Z') || + (runeValue >= 'a' && runeValue <= 'z'): + thisdir := dir%25 + 1 + pos := int(runeValue - 'A') + if pos >= 26 { + pos -= 6 + } + pos = pos - thisdir + if pos < 0 { + pos += 52 + } + if pos >= 26 { + pos += 6 + } + _, _ = result.WriteRune(rune('A' + pos)) + + case runeValue >= 0xA0 && runeValue <= 0xFF: + thisdir := (dir % 95) + 1 + newRune := 0xA0 + int(runeValue) - 0xA0 - thisdir + if newRune < 0xA0 { + newRune += 96 + } + _, _ = result.WriteRune(rune(newRune)) + + case runeValue >= 0x100: + thisdir := (dir % 127) + 1 + base := int(runeValue - runeValue%256) + newRune := rune(base + (int(runeValue) - base - thisdir)) + if int(newRune) < base { + newRune += 256 + } + _, _ = result.WriteRune(rune(newRune)) + + default: + _, _ = result.WriteRune(runeValue) + + } + } + + return result.String(), nil +} + +// encryptFileName encrypts a file path +func (c *cipher) encryptFileName(in string) string { + segments := strings.Split(in, "/") + for i := range segments { + // Skip directory name encryption if the user chose to + // leave them intact + if !c.dirNameEncrypt && i != (len(segments)-1) { + continue + } + if c.mode == NameEncryptionStandard { + segments[i] = c.encryptSegment(segments[i]) + } else { + segments[i] = c.obfuscateSegment(segments[i]) + } + } + return strings.Join(segments, "/") +} + +// EncryptFileName encrypts a file path +func (c *cipher) EncryptFileName(in string) string { + if c.mode == NameEncryptionOff { + return in + encryptedSuffix + } + return c.encryptFileName(in) +} + +// EncryptDirName encrypts a directory path +func (c *cipher) EncryptDirName(in string) string { + if c.mode == NameEncryptionOff || !c.dirNameEncrypt { + return in + } + return c.encryptFileName(in) +} + +// decryptFileName decrypts a file path +func (c *cipher) decryptFileName(in string) (string, error) { + segments := strings.Split(in, "/") + for i := range segments { + var err error + // Skip directory name decryption if the user chose to + // leave them intact + if !c.dirNameEncrypt && i != (len(segments)-1) { + continue + } + if c.mode == NameEncryptionStandard { + segments[i], err = c.decryptSegment(segments[i]) + } else { + segments[i], err = c.deobfuscateSegment(segments[i]) + } + + if err != nil { + return "", err + } + } + return strings.Join(segments, "/"), nil +} + +// DecryptFileName decrypts a file path +func (c *cipher) DecryptFileName(in string) (string, error) { + if c.mode == NameEncryptionOff { + remainingLength := len(in) - len(encryptedSuffix) + if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) { + return in[:remainingLength], nil + } + return "", ErrorNotAnEncryptedFile + } + return c.decryptFileName(in) +} + +// DecryptDirName decrypts a directory path +func (c *cipher) DecryptDirName(in string) (string, error) { + if c.mode == NameEncryptionOff || !c.dirNameEncrypt { + return in, nil + } + return c.decryptFileName(in) +} + +func (c *cipher) NameEncryptionMode() NameEncryptionMode { + return c.mode +} + +// nonce is an NACL secretbox nonce +type nonce [fileNonceSize]byte + +// pointer returns the nonce as a *[24]byte for secretbox +func (n *nonce) pointer() *[fileNonceSize]byte { + return (*[fileNonceSize]byte)(n) +} + +// fromReader fills the nonce from an io.Reader - normally the OSes +// crypto random number generator +func (n *nonce) fromReader(in io.Reader) error { + read, err := io.ReadFull(in, (*n)[:]) + if read != fileNonceSize { + return errors.Wrap(err, "short read of nonce") + } + return nil +} + +// fromBuf fills the nonce from the buffer passed in +func (n *nonce) fromBuf(buf []byte) { + read := copy((*n)[:], buf) + if read != fileNonceSize { + panic("buffer to short to read nonce") + } +} + +// carry 1 up the nonce from position i +func (n *nonce) carry(i int) { + for ; i < len(*n); i++ { + digit := (*n)[i] + newDigit := digit + 1 + (*n)[i] = newDigit + if newDigit >= digit { + // exit if no carry + break + } + } +} + +// increment to add 1 to the nonce +func (n *nonce) increment() { + n.carry(0) +} + +// add an uint64 to the nonce +func (n *nonce) add(x uint64) { + carry := uint16(0) + for i := 0; i < 8; i++ { + digit := (*n)[i] + xDigit := byte(x) + x >>= 8 + carry += uint16(digit) + uint16(xDigit) + (*n)[i] = byte(carry) + carry >>= 8 + } + if carry != 0 { + n.carry(8) + } +} + +// encrypter encrypts an io.Reader on the fly +type encrypter struct { + mu sync.Mutex + in io.Reader + c *cipher + nonce nonce + buf []byte + readBuf []byte + bufIndex int + bufSize int + err error +} + +// newEncrypter creates a new file handle encrypting on the fly +func (c *cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) { + fh := &encrypter{ + in: in, + c: c, + buf: c.getBlock(), + readBuf: c.getBlock(), + bufSize: fileHeaderSize, + } + // Initialise nonce + if nonce != nil { + fh.nonce = *nonce + } else { + err := fh.nonce.fromReader(c.cryptoRand) + if err != nil { + return nil, err + } + } + // Copy magic into buffer + copy(fh.buf, fileMagicBytes) + // Copy nonce into buffer + copy(fh.buf[fileMagicSize:], fh.nonce[:]) + return fh, nil +} + +// Read as per io.Reader +func (fh *encrypter) Read(p []byte) (n int, err error) { + fh.mu.Lock() + defer fh.mu.Unlock() + + if fh.err != nil { + return 0, fh.err + } + if fh.bufIndex >= fh.bufSize { + // Read data + // FIXME should overlap the reads with a go-routine and 2 buffers? + readBuf := fh.readBuf[:blockDataSize] + n, err = io.ReadFull(fh.in, readBuf) + if n == 0 { + // err can't be nil since: + // n == len(buf) if and only if err == nil. + return fh.finish(err) + } + // possibly err != nil here, but we will process the + // data and the next call to ReadFull will return 0, err + // Write nonce to start of block + copy(fh.buf, fh.nonce[:]) + // Encrypt the block using the nonce + block := fh.buf + secretbox.Seal(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey) + fh.bufIndex = 0 + fh.bufSize = blockHeaderSize + n + fh.nonce.increment() + } + n = copy(p, fh.buf[fh.bufIndex:fh.bufSize]) + fh.bufIndex += n + return n, nil +} + +// finish sets the final error and tidies up +func (fh *encrypter) finish(err error) (int, error) { + if fh.err != nil { + return 0, fh.err + } + fh.err = err + fh.c.putBlock(fh.buf) + fh.buf = nil + fh.c.putBlock(fh.readBuf) + fh.readBuf = nil + return 0, err +} + +// Encrypt data encrypts the data stream +func (c *cipher) EncryptData(in io.Reader) (io.Reader, error) { + in, wrap := accounting.UnWrap(in) // unwrap the accounting off the Reader + out, err := c.newEncrypter(in, nil) + if err != nil { + return nil, err + } + return wrap(out), nil // and wrap the accounting back on +} + +// decrypter decrypts an io.ReaderCloser on the fly +type decrypter struct { + mu sync.Mutex + rc io.ReadCloser + nonce nonce + initialNonce nonce + c *cipher + buf []byte + readBuf []byte + bufIndex int + bufSize int + err error + limit int64 // limit of bytes to read, -1 for unlimited + open OpenRangeSeek +} + +// newDecrypter creates a new file handle decrypting on the fly +func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) { + fh := &decrypter{ + rc: rc, + c: c, + buf: c.getBlock(), + readBuf: c.getBlock(), + limit: -1, + } + // Read file header (magic + nonce) + readBuf := fh.readBuf[:fileHeaderSize] + _, err := io.ReadFull(fh.rc, readBuf) + if err == io.EOF || err == io.ErrUnexpectedEOF { + // This read from 0..fileHeaderSize-1 bytes + return nil, fh.finishAndClose(ErrorEncryptedFileTooShort) + } else if err != nil { + return nil, fh.finishAndClose(err) + } + // check the magic + if !bytes.Equal(readBuf[:fileMagicSize], fileMagicBytes) { + return nil, fh.finishAndClose(ErrorEncryptedBadMagic) + } + // retreive the nonce + fh.nonce.fromBuf(readBuf[fileMagicSize:]) + fh.initialNonce = fh.nonce + return fh, nil +} + +// newDecrypterSeek creates a new file handle decrypting on the fly +func (c *cipher) newDecrypterSeek(open OpenRangeSeek, offset, limit int64) (fh *decrypter, err error) { + var rc io.ReadCloser + doRangeSeek := false + setLimit := false + // Open initially with no seek + if offset == 0 && limit < 0 { + // If no offset or limit then open whole file + rc, err = open(0, -1) + } else if offset == 0 { + // If no offset open the header + limit worth of the file + _, underlyingLimit, _, _ := calculateUnderlying(offset, limit) + rc, err = open(0, int64(fileHeaderSize)+underlyingLimit) + setLimit = true + } else { + // Otherwise just read the header to start with + rc, err = open(0, int64(fileHeaderSize)) + doRangeSeek = true + } + if err != nil { + return nil, err + } + // Open the stream which fills in the nonce + fh, err = c.newDecrypter(rc) + if err != nil { + return nil, err + } + fh.open = open // will be called by fh.RangeSeek + if doRangeSeek { + _, err = fh.RangeSeek(offset, io.SeekStart, limit) + if err != nil { + _ = fh.Close() + return nil, err + } + } + if setLimit { + fh.limit = limit + } + return fh, nil +} + +// read data into internal buffer - call with fh.mu held +func (fh *decrypter) fillBuffer() (err error) { + // FIXME should overlap the reads with a go-routine and 2 buffers? + readBuf := fh.readBuf + n, err := io.ReadFull(fh.rc, readBuf) + if n == 0 { + // err can't be nil since: + // n == len(buf) if and only if err == nil. + return err + } + // possibly err != nil here, but we will process the data and + // the next call to ReadFull will return 0, err + + // Check header + 1 byte exists + if n <= blockHeaderSize { + if err != nil { + return err // return pending error as it is likely more accurate + } + return ErrorEncryptedFileBadHeader + } + // Decrypt the block using the nonce + block := fh.buf + _, ok := secretbox.Open(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey) + if !ok { + if err != nil { + return err // return pending error as it is likely more accurate + } + return ErrorEncryptedBadBlock + } + fh.bufIndex = 0 + fh.bufSize = n - blockHeaderSize + fh.nonce.increment() + return nil +} + +// Read as per io.Reader +func (fh *decrypter) Read(p []byte) (n int, err error) { + fh.mu.Lock() + defer fh.mu.Unlock() + + if fh.err != nil { + return 0, fh.err + } + if fh.bufIndex >= fh.bufSize { + err = fh.fillBuffer() + if err != nil { + return 0, fh.finish(err) + } + } + toCopy := fh.bufSize - fh.bufIndex + if fh.limit >= 0 && fh.limit < int64(toCopy) { + toCopy = int(fh.limit) + } + n = copy(p, fh.buf[fh.bufIndex:fh.bufIndex+toCopy]) + fh.bufIndex += n + if fh.limit >= 0 { + fh.limit -= int64(n) + if fh.limit == 0 { + return n, fh.finish(io.EOF) + } + } + return n, nil +} + +// calculateUnderlying converts an (offset, limit) in a crypted file +// into an (underlyingOffset, underlyingLimit) for the underlying +// file. +// +// It also returns number of bytes to discard after reading the first +// block and number of blocks this is from the start so the nonce can +// be incremented. +func calculateUnderlying(offset, limit int64) (underlyingOffset, underlyingLimit, discard, blocks int64) { + // blocks we need to seek, plus bytes we need to discard + blocks, discard = offset/blockDataSize, offset%blockDataSize + + // Offset in underlying stream we need to seek + underlyingOffset = int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize) + + // work out how many blocks we need to read + underlyingLimit = int64(-1) + if limit >= 0 { + // bytes to read beyond the first block + bytesToRead := limit - (blockDataSize - discard) + + // Read the first block + blocksToRead := int64(1) + + if bytesToRead > 0 { + // Blocks that need to be read plus left over blocks + extraBlocksToRead, endBytes := bytesToRead/blockDataSize, bytesToRead%blockDataSize + if endBytes != 0 { + // If left over bytes must read another block + extraBlocksToRead++ + } + blocksToRead += extraBlocksToRead + } + + // Must read a whole number of blocks + underlyingLimit = blocksToRead * (blockHeaderSize + blockDataSize) + } + return +} + +// RangeSeek behaves like a call to Seek(offset int64, whence +// int) with the output wrapped in an io.LimitedReader +// limiting the total length to limit. +// +// RangeSeek with a limit of < 0 is equivalent to a regular Seek. +func (fh *decrypter) RangeSeek(offset int64, whence int, limit int64) (int64, error) { + fh.mu.Lock() + defer fh.mu.Unlock() + + if fh.open == nil { + return 0, fh.finish(errors.New("can't seek - not initialised with newDecrypterSeek")) + } + if whence != io.SeekStart { + return 0, fh.finish(errors.New("can only seek from the start")) + } + + // Reset error or return it if not EOF + if fh.err == io.EOF { + fh.unFinish() + } else if fh.err != nil { + return 0, fh.err + } + + underlyingOffset, underlyingLimit, discard, blocks := calculateUnderlying(offset, limit) + + // Move the nonce on the correct number of blocks from the start + fh.nonce = fh.initialNonce + fh.nonce.add(uint64(blocks)) + + // Can we seek underlying stream directly? + if do, ok := fh.rc.(fs.RangeSeeker); ok { + // Seek underlying stream directly + _, err := do.RangeSeek(underlyingOffset, 0, underlyingLimit) + if err != nil { + return 0, fh.finish(err) + } + } else { + // if not reopen with seek + _ = fh.rc.Close() // close underlying file + fh.rc = nil + + // Re-open the underlying object with the offset given + rc, err := fh.open(underlyingOffset, underlyingLimit) + if err != nil { + return 0, fh.finish(errors.Wrap(err, "couldn't reopen file with offset and limit")) + } + + // Set the file handle + fh.rc = rc + } + + // Fill the buffer + err := fh.fillBuffer() + if err != nil { + return 0, fh.finish(err) + } + + // Discard bytes from the buffer + if int(discard) > fh.bufSize { + return 0, fh.finish(ErrorBadSeek) + } + fh.bufIndex = int(discard) + + // Set the limit + fh.limit = limit + + return offset, nil +} + +// Seek implements the io.Seeker interface +func (fh *decrypter) Seek(offset int64, whence int) (int64, error) { + return fh.RangeSeek(offset, whence, -1) +} + +// finish sets the final error and tidies up +func (fh *decrypter) finish(err error) error { + if fh.err != nil { + return fh.err + } + fh.err = err + fh.c.putBlock(fh.buf) + fh.buf = nil + fh.c.putBlock(fh.readBuf) + fh.readBuf = nil + return err +} + +// unFinish undoes the effects of finish +func (fh *decrypter) unFinish() { + // Clear error + fh.err = nil + + // reinstate the buffers + fh.buf = fh.c.getBlock() + fh.readBuf = fh.c.getBlock() + + // Empty the buffer + fh.bufIndex = 0 + fh.bufSize = 0 +} + +// Close +func (fh *decrypter) Close() error { + fh.mu.Lock() + defer fh.mu.Unlock() + + // Check already closed + if fh.err == ErrorFileClosed { + return fh.err + } + // Closed before reading EOF so not finish()ed yet + if fh.err == nil { + _ = fh.finish(io.EOF) + } + // Show file now closed + fh.err = ErrorFileClosed + if fh.rc == nil { + return nil + } + return fh.rc.Close() +} + +// finishAndClose does finish then Close() +// +// Used when we are returning a nil fh from new +func (fh *decrypter) finishAndClose(err error) error { + _ = fh.finish(err) + _ = fh.Close() + return err +} + +// DecryptData decrypts the data stream +func (c *cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) { + out, err := c.newDecrypter(rc) + if err != nil { + return nil, err + } + return out, nil +} + +// DecryptDataSeek decrypts the data stream from offset +// +// The open function must return a ReadCloser opened to the offset supplied +// +// You must use this form of DecryptData if you might want to Seek the file handle +func (c *cipher) DecryptDataSeek(open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) { + out, err := c.newDecrypterSeek(open, offset, limit) + if err != nil { + return nil, err + } + return out, nil +} + +// EncryptedSize calculates the size of the data when encrypted +func (c *cipher) EncryptedSize(size int64) int64 { + blocks, residue := size/blockDataSize, size%blockDataSize + encryptedSize := int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize) + if residue != 0 { + encryptedSize += blockHeaderSize + residue + } + return encryptedSize +} + +// DecryptedSize calculates the size of the data when decrypted +func (c *cipher) DecryptedSize(size int64) (int64, error) { + size -= int64(fileHeaderSize) + if size < 0 { + return 0, ErrorEncryptedFileTooShort + } + blocks, residue := size/blockSize, size%blockSize + decryptedSize := blocks * blockDataSize + if residue != 0 { + residue -= blockHeaderSize + if residue <= 0 { + return 0, ErrorEncryptedFileBadHeader + } + } + decryptedSize += residue + return decryptedSize, nil +} + +// check interfaces +var ( + _ Cipher = (*cipher)(nil) + _ io.ReadCloser = (*decrypter)(nil) + _ io.Seeker = (*decrypter)(nil) + _ fs.RangeSeeker = (*decrypter)(nil) + _ io.Reader = (*encrypter)(nil) +) diff --git a/.rclone_repo/backend/crypt/cipher_test.go b/.rclone_repo/backend/crypt/cipher_test.go new file mode 100755 index 0000000..601d1e7 --- /dev/null +++ b/.rclone_repo/backend/crypt/cipher_test.go @@ -0,0 +1,1290 @@ +package crypt + +import ( + "bytes" + "encoding/base32" + "fmt" + "io" + "io/ioutil" + "strings" + "testing" + + "github.com/ncw/rclone/backend/crypt/pkcs7" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewNameEncryptionMode(t *testing.T) { + for _, test := range []struct { + in string + expected NameEncryptionMode + expectedErr string + }{ + {"off", NameEncryptionOff, ""}, + {"standard", NameEncryptionStandard, ""}, + {"obfuscate", NameEncryptionObfuscated, ""}, + {"potato", NameEncryptionOff, "Unknown file name encryption mode \"potato\""}, + } { + actual, actualErr := NewNameEncryptionMode(test.in) + assert.Equal(t, actual, test.expected) + if test.expectedErr == "" { + assert.NoError(t, actualErr) + } else { + assert.Error(t, actualErr, test.expectedErr) + } + } +} + +func TestNewNameEncryptionModeString(t *testing.T) { + assert.Equal(t, NameEncryptionOff.String(), "off") + assert.Equal(t, NameEncryptionStandard.String(), "standard") + assert.Equal(t, NameEncryptionObfuscated.String(), "obfuscate") + assert.Equal(t, NameEncryptionMode(3).String(), "Unknown mode #3") +} + +func TestValidString(t *testing.T) { + for _, test := range []struct { + in string + expected error + }{ + {"", nil}, + {"\x01", ErrorBadDecryptControlChar}, + {"a\x02", ErrorBadDecryptControlChar}, + {"abc\x03", ErrorBadDecryptControlChar}, + {"abc\x04def", ErrorBadDecryptControlChar}, + {"\x05d", ErrorBadDecryptControlChar}, + {"\x06def", ErrorBadDecryptControlChar}, + {"\x07", ErrorBadDecryptControlChar}, + {"\x08", ErrorBadDecryptControlChar}, + {"\x09", ErrorBadDecryptControlChar}, + {"\x0A", ErrorBadDecryptControlChar}, + {"\x0B", ErrorBadDecryptControlChar}, + {"\x0C", ErrorBadDecryptControlChar}, + {"\x0D", ErrorBadDecryptControlChar}, + {"\x0E", ErrorBadDecryptControlChar}, + {"\x0F", ErrorBadDecryptControlChar}, + {"\x10", ErrorBadDecryptControlChar}, + {"\x11", ErrorBadDecryptControlChar}, + {"\x12", ErrorBadDecryptControlChar}, + {"\x13", ErrorBadDecryptControlChar}, + {"\x14", ErrorBadDecryptControlChar}, + {"\x15", ErrorBadDecryptControlChar}, + {"\x16", ErrorBadDecryptControlChar}, + {"\x17", ErrorBadDecryptControlChar}, + {"\x18", ErrorBadDecryptControlChar}, + {"\x19", ErrorBadDecryptControlChar}, + {"\x1A", ErrorBadDecryptControlChar}, + {"\x1B", ErrorBadDecryptControlChar}, + {"\x1C", ErrorBadDecryptControlChar}, + {"\x1D", ErrorBadDecryptControlChar}, + {"\x1E", ErrorBadDecryptControlChar}, + {"\x1F", ErrorBadDecryptControlChar}, + {"\x20", nil}, + {"\x7E", nil}, + {"\x7F", ErrorBadDecryptControlChar}, + {"£100", nil}, + {`hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`, nil}, + {"£100", nil}, + // Following tests from https://secure.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805 + {"a", nil}, // Valid ASCII + {"\xc3\xb1", nil}, // Valid 2 Octet Sequence + {"\xc3\x28", ErrorBadDecryptUTF8}, // Invalid 2 Octet Sequence + {"\xa0\xa1", ErrorBadDecryptUTF8}, // Invalid Sequence Identifier + {"\xe2\x82\xa1", nil}, // Valid 3 Octet Sequence + {"\xe2\x28\xa1", ErrorBadDecryptUTF8}, // Invalid 3 Octet Sequence (in 2nd Octet) + {"\xe2\x82\x28", ErrorBadDecryptUTF8}, // Invalid 3 Octet Sequence (in 3rd Octet) + {"\xf0\x90\x8c\xbc", nil}, // Valid 4 Octet Sequence + {"\xf0\x28\x8c\xbc", ErrorBadDecryptUTF8}, // Invalid 4 Octet Sequence (in 2nd Octet) + {"\xf0\x90\x28\xbc", ErrorBadDecryptUTF8}, // Invalid 4 Octet Sequence (in 3rd Octet) + {"\xf0\x28\x8c\x28", ErrorBadDecryptUTF8}, // Invalid 4 Octet Sequence (in 4th Octet) + {"\xf8\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8}, // Valid 5 Octet Sequence (but not Unicode!) + {"\xfc\xa1\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8}, // Valid 6 Octet Sequence (but not Unicode!) + } { + actual := checkValidString([]byte(test.in)) + assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in)) + } +} + +func TestEncodeFileName(t *testing.T) { + for _, test := range []struct { + in string + expected string + }{ + {"", ""}, + {"1", "64"}, + {"12", "64p0"}, + {"123", "64p36"}, + {"1234", "64p36d0"}, + {"12345", "64p36d1l"}, + {"123456", "64p36d1l6o"}, + {"1234567", "64p36d1l6org"}, + {"12345678", "64p36d1l6orjg"}, + {"123456789", "64p36d1l6orjge8"}, + {"1234567890", "64p36d1l6orjge9g"}, + {"12345678901", "64p36d1l6orjge9g64"}, + {"123456789012", "64p36d1l6orjge9g64p0"}, + {"1234567890123", "64p36d1l6orjge9g64p36"}, + {"12345678901234", "64p36d1l6orjge9g64p36d0"}, + {"123456789012345", "64p36d1l6orjge9g64p36d1l"}, + {"1234567890123456", "64p36d1l6orjge9g64p36d1l6o"}, + } { + actual := encodeFileName([]byte(test.in)) + assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in)) + recovered, err := decodeFileName(test.expected) + assert.NoError(t, err) + assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", test.expected)) + in := strings.ToUpper(test.expected) + recovered, err = decodeFileName(in) + assert.NoError(t, err) + assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", in)) + } +} + +func TestDecodeFileName(t *testing.T) { + // We've tested decoding the valid ones above, now concentrate on the invalid ones + for _, test := range []struct { + in string + expectedErr error + }{ + {"64=", ErrorBadBase32Encoding}, + {"!", base32.CorruptInputError(0)}, + {"hello=hello", base32.CorruptInputError(5)}, + } { + actual, actualErr := decodeFileName(test.in) + assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr)) + } +} + +func TestEncryptSegment(t *testing.T) { + c, _ := newCipher(NameEncryptionStandard, "", "", true) + for _, test := range []struct { + in string + expected string + }{ + {"", ""}, + {"1", "p0e52nreeaj0a5ea7s64m4j72s"}, + {"12", "l42g6771hnv3an9cgc8cr2n1ng"}, + {"123", "qgm4avr35m5loi1th53ato71v0"}, + {"1234", "8ivr2e9plj3c3esisjpdisikos"}, + {"12345", "rh9vu63q3o29eqmj4bg6gg7s44"}, + {"123456", "bn717l3alepn75b2fb2ejmi4b4"}, + {"1234567", "n6bo9jmb1qe3b1ogtj5qkf19k8"}, + {"12345678", "u9t24j7uaq94dh5q53m3s4t9ok"}, + {"123456789", "37hn305g6j12d1g0kkrl7ekbs4"}, + {"1234567890", "ot8d91eplaglb62k2b1trm2qv0"}, + {"12345678901", "h168vvrgb53qnrtvvmb378qrcs"}, + {"123456789012", "s3hsdf9e29ithrqbjqu01t8q2s"}, + {"1234567890123", "cf3jimlv1q2oc553mv7s3mh3eo"}, + {"12345678901234", "moq0uqdlqrblrc5pa5u5c7hq9g"}, + {"123456789012345", "eeam3li4rnommi3a762h5n7meg"}, + {"1234567890123456", "mijbj0frqf6ms7frcr6bd9h0env53jv96pjaaoirk7forcgpt70g"}, + } { + actual := c.encryptSegment(test.in) + assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %q", test.in)) + recovered, err := c.decryptSegment(test.expected) + assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected)) + assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected)) + in := strings.ToUpper(test.expected) + recovered, err = c.decryptSegment(in) + assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", in)) + assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", in)) + } +} + +func TestDecryptSegment(t *testing.T) { + // We've tested the forwards above, now concentrate on the errors + c, _ := newCipher(NameEncryptionStandard, "", "", true) + for _, test := range []struct { + in string + expectedErr error + }{ + {"64=", ErrorBadBase32Encoding}, + {"!", base32.CorruptInputError(0)}, + {encodeFileName([]byte("a")), ErrorNotAMultipleOfBlocksize}, + {encodeFileName([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize}, + {encodeFileName([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong}, + {c.encryptSegment("\x01"), ErrorBadDecryptControlChar}, + {c.encryptSegment("\xc3\x28"), ErrorBadDecryptUTF8}, + } { + actual, actualErr := c.decryptSegment(test.in) + assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr)) + } +} + +func TestEncryptFileName(t *testing.T) { + // First standard mode + c, _ := newCipher(NameEncryptionStandard, "", "", true) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1")) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12")) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123")) + // Standard mode with directory name encryption off + c, _ = newCipher(NameEncryptionStandard, "", "", false) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1")) + assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12")) + assert.Equal(t, "1/12/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123")) + // Now off mode + c, _ = newCipher(NameEncryptionOff, "", "", true) + assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123")) + // Obfuscation mode + c, _ = newCipher(NameEncryptionObfuscated, "", "", true) + assert.Equal(t, "49.6/99.23/150.890/53.!!lipps", c.EncryptFileName("1/12/123/!hello")) + assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1")) + assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0")) + // Obfuscation mode with directory name encryption off + c, _ = newCipher(NameEncryptionObfuscated, "", "", false) + assert.Equal(t, "1/12/123/53.!!lipps", c.EncryptFileName("1/12/123/!hello")) + assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1")) + assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0")) +} + +func TestDecryptFileName(t *testing.T) { + for _, test := range []struct { + mode NameEncryptionMode + dirNameEncrypt bool + in string + expected string + expectedErr error + }{ + {NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil}, + {NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil}, + {NameEncryptionStandard, true, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil}, + {NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil}, + {NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize}, + {NameEncryptionStandard, false, "1/12/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil}, + {NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil}, + {NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile}, + {NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile}, + {NameEncryptionObfuscated, true, "!.hello", "hello", nil}, + {NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile}, + {NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil}, + {NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil}, + {NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil}, + } { + c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt) + actual, actualErr := c.DecryptFileName(test.in) + what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode) + assert.Equal(t, test.expected, actual, what) + assert.Equal(t, test.expectedErr, actualErr, what) + } +} + +func TestEncDecMatches(t *testing.T) { + for _, test := range []struct { + mode NameEncryptionMode + in string + }{ + {NameEncryptionStandard, "1/2/3/4"}, + {NameEncryptionOff, "1/2/3/4"}, + {NameEncryptionObfuscated, "1/2/3/4/!hello\u03a0"}, + {NameEncryptionObfuscated, "Avatar The Last Airbender"}, + } { + c, _ := newCipher(test.mode, "", "", true) + out, err := c.DecryptFileName(c.EncryptFileName(test.in)) + what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode) + assert.Equal(t, out, test.in, what) + assert.Equal(t, err, nil, what) + } +} + +func TestEncryptDirName(t *testing.T) { + // First standard mode + c, _ := newCipher(NameEncryptionStandard, "", "", true) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptDirName("1")) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptDirName("1/12")) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptDirName("1/12/123")) + // Standard mode with dir name encryption off + c, _ = newCipher(NameEncryptionStandard, "", "", false) + assert.Equal(t, "1/12", c.EncryptDirName("1/12")) + assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123")) + // Now off mode + c, _ = newCipher(NameEncryptionOff, "", "", true) + assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123")) +} + +func TestDecryptDirName(t *testing.T) { + for _, test := range []struct { + mode NameEncryptionMode + dirNameEncrypt bool + in string + expected string + expectedErr error + }{ + {NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil}, + {NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil}, + {NameEncryptionStandard, true, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil}, + {NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil}, + {NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize}, + {NameEncryptionStandard, false, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", nil}, + {NameEncryptionStandard, false, "1/12/123", "1/12/123", nil}, + {NameEncryptionOff, true, "1/12/123.bin", "1/12/123.bin", nil}, + {NameEncryptionOff, true, "1/12/123", "1/12/123", nil}, + {NameEncryptionOff, true, ".bin", ".bin", nil}, + } { + c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt) + actual, actualErr := c.DecryptDirName(test.in) + what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode) + assert.Equal(t, test.expected, actual, what) + assert.Equal(t, test.expectedErr, actualErr, what) + } +} + +func TestEncryptedSize(t *testing.T) { + c, _ := newCipher(NameEncryptionStandard, "", "", true) + for _, test := range []struct { + in int64 + expected int64 + }{ + {0, 32}, + {1, 32 + 16 + 1}, + {65536, 32 + 16 + 65536}, + {65537, 32 + 16 + 65536 + 16 + 1}, + {1 << 20, 32 + 16*(16+65536)}, + {(1 << 20) + 65535, 32 + 16*(16+65536) + 16 + 65535}, + {1 << 30, 32 + 16384*(16+65536)}, + {(1 << 40) + 1, 32 + 16777216*(16+65536) + 16 + 1}, + } { + actual := c.EncryptedSize(test.in) + assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %d", test.in)) + recovered, err := c.DecryptedSize(test.expected) + assert.NoError(t, err, fmt.Sprintf("Testing reverse %d", test.expected)) + assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %d", test.expected)) + } +} + +func TestDecryptedSize(t *testing.T) { + // Test the errors since we tested the reverse above + c, _ := newCipher(NameEncryptionStandard, "", "", true) + for _, test := range []struct { + in int64 + expectedErr error + }{ + {0, ErrorEncryptedFileTooShort}, + {0, ErrorEncryptedFileTooShort}, + {1, ErrorEncryptedFileTooShort}, + {7, ErrorEncryptedFileTooShort}, + {32 + 1, ErrorEncryptedFileBadHeader}, + {32 + 16, ErrorEncryptedFileBadHeader}, + {32 + 16 + 65536 + 1, ErrorEncryptedFileBadHeader}, + {32 + 16 + 65536 + 16, ErrorEncryptedFileBadHeader}, + } { + _, actualErr := c.DecryptedSize(test.in) + assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("Testing %d", test.in)) + } +} + +func TestNoncePointer(t *testing.T) { + var x nonce + assert.Equal(t, (*[24]byte)(&x), x.pointer()) +} + +func TestNonceFromReader(t *testing.T) { + var x nonce + buf := bytes.NewBufferString("123456789abcdefghijklmno") + err := x.fromReader(buf) + assert.NoError(t, err) + assert.Equal(t, nonce{'1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'}, x) + buf = bytes.NewBufferString("123456789abcdefghijklmn") + err = x.fromReader(buf) + assert.Error(t, err, "short read of nonce") +} + +func TestNonceFromBuf(t *testing.T) { + var x nonce + buf := []byte("123456789abcdefghijklmnoXXXXXXXX") + x.fromBuf(buf) + assert.Equal(t, nonce{'1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'}, x) + buf = []byte("0123456789abcdefghijklmn") + x.fromBuf(buf) + assert.Equal(t, nonce{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n'}, x) + buf = []byte("0123456789abcdefghijklm") + assert.Panics(t, func() { x.fromBuf(buf) }) +} + +func TestNonceIncrement(t *testing.T) { + for _, test := range []struct { + in nonce + out nonce + }{ + { + nonce{0x00}, + nonce{0x01}, + }, + { + nonce{0xFF}, + nonce{0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF}, + nonce{0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + }, + } { + x := test.in + x.increment() + assert.Equal(t, test.out, x) + } +} + +func TestNonceAdd(t *testing.T) { + for _, test := range []struct { + add uint64 + in nonce + out nonce + }{ + { + 0x01, + nonce{0x00}, + nonce{0x01}, + }, + { + 0xFF, + nonce{0xFF}, + nonce{0xFE, 0x01}, + }, + { + 0xFFFF, + nonce{0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0x01}, + }, + { + 0xFFFFFF, + nonce{0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFe, 0xFF, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + }, + } { + x := test.in + x.add(test.add) + assert.Equal(t, test.out, x) + } +} + +// randomSource can read or write a random sequence +type randomSource struct { + counter int64 + size int64 +} + +func newRandomSource(size int64) *randomSource { + return &randomSource{ + size: size, + } +} + +func (r *randomSource) next() byte { + r.counter++ + return byte(r.counter % 257) +} + +func (r *randomSource) Read(p []byte) (n int, err error) { + for i := range p { + if r.counter >= r.size { + err = io.EOF + break + } + p[i] = r.next() + n++ + } + return n, err +} + +func (r *randomSource) Write(p []byte) (n int, err error) { + for i := range p { + if p[i] != r.next() { + return 0, errors.Errorf("Error in stream at %d", r.counter) + } + } + return len(p), nil +} + +func (r *randomSource) Close() error { return nil } + +// Check interfaces +var ( + _ io.ReadCloser = (*randomSource)(nil) + _ io.WriteCloser = (*randomSource)(nil) +) + +// Test test infrastructure first! +func TestRandomSource(t *testing.T) { + source := newRandomSource(1E8) + sink := newRandomSource(1E8) + n, err := io.Copy(sink, source) + assert.NoError(t, err) + assert.Equal(t, int64(1E8), n) + + source = newRandomSource(1E8) + buf := make([]byte, 16) + _, _ = source.Read(buf) + sink = newRandomSource(1E8) + _, err = io.Copy(sink, source) + assert.Error(t, err, "Error in stream") +} + +type zeroes struct{} + +func (z *zeroes) Read(p []byte) (n int, err error) { + for i := range p { + p[i] = 0 + n++ + } + return n, nil +} + +// Test encrypt decrypt with different buffer sizes +func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) { + c, err := newCipher(NameEncryptionStandard, "", "", true) + assert.NoError(t, err) + c.cryptoRand = &zeroes{} // zero out the nonce + buf := make([]byte, bufSize) + source := newRandomSource(copySize) + encrypted, err := c.newEncrypter(source, nil) + assert.NoError(t, err) + decrypted, err := c.newDecrypter(ioutil.NopCloser(encrypted)) + assert.NoError(t, err) + sink := newRandomSource(copySize) + n, err := io.CopyBuffer(sink, decrypted, buf) + assert.NoError(t, err) + assert.Equal(t, copySize, n) + blocks := copySize / blockSize + if (copySize % blockSize) != 0 { + blocks++ + } + var expectedNonce = nonce{byte(blocks), byte(blocks >> 8), byte(blocks >> 16), byte(blocks >> 32)} + assert.Equal(t, expectedNonce, encrypted.nonce) + assert.Equal(t, expectedNonce, decrypted.nonce) +} + +func TestEncryptDecrypt1(t *testing.T) { + testEncryptDecrypt(t, 1, 1E7) +} + +func TestEncryptDecrypt32(t *testing.T) { + testEncryptDecrypt(t, 32, 1E8) +} + +func TestEncryptDecrypt4096(t *testing.T) { + testEncryptDecrypt(t, 4096, 1E8) +} + +func TestEncryptDecrypt65536(t *testing.T) { + testEncryptDecrypt(t, 65536, 1E8) +} + +func TestEncryptDecrypt65537(t *testing.T) { + testEncryptDecrypt(t, 65537, 1E8) +} + +var ( + file0 = []byte{ + 0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + } + file1 = []byte{ + 0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x09, 0x5b, 0x44, 0x6c, 0xd6, 0x23, 0x7b, 0xbc, 0xb0, 0x8d, 0x09, 0xfb, 0x52, 0x4c, 0xe5, 0x65, + 0xAA, + } + file16 = []byte{ + 0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0xb9, 0xc4, 0x55, 0x2a, 0x27, 0x10, 0x06, 0x29, 0x18, 0x96, 0x0a, 0x3e, 0x60, 0x8c, 0x29, 0xb9, + 0xaa, 0x8a, 0x5e, 0x1e, 0x16, 0x5b, 0x6d, 0x07, 0x5d, 0xe4, 0xe9, 0xbb, 0x36, 0x7f, 0xd6, 0xd4, + } +) + +func TestEncryptData(t *testing.T) { + for _, test := range []struct { + in []byte + expected []byte + }{ + {[]byte{}, file0}, + {[]byte{1}, file1}, + {[]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, file16}, + } { + c, err := newCipher(NameEncryptionStandard, "", "", true) + assert.NoError(t, err) + c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator + + // Check encode works + buf := bytes.NewBuffer(test.in) + encrypted, err := c.EncryptData(buf) + assert.NoError(t, err) + out, err := ioutil.ReadAll(encrypted) + assert.NoError(t, err) + assert.Equal(t, test.expected, out) + + // Check we can decode the data properly too... + buf = bytes.NewBuffer(out) + decrypted, err := c.DecryptData(ioutil.NopCloser(buf)) + assert.NoError(t, err) + out, err = ioutil.ReadAll(decrypted) + assert.NoError(t, err) + assert.Equal(t, test.in, out) + } +} + +func TestNewEncrypter(t *testing.T) { + c, err := newCipher(NameEncryptionStandard, "", "", true) + assert.NoError(t, err) + c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator + + z := &zeroes{} + + fh, err := c.newEncrypter(z, nil) + assert.NoError(t, err) + assert.Equal(t, nonce{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.nonce) + assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.buf[:32]) + + // Test error path + c.cryptoRand = bytes.NewBufferString("123456789abcdefghijklmn") + fh, err = c.newEncrypter(z, nil) + assert.Nil(t, fh) + assert.Error(t, err, "short read of nonce") + +} + +// Test the stream returning 0, io.ErrUnexpectedEOF - this used to +// cause a fatal loop +func TestNewEncrypterErrUnexpectedEOF(t *testing.T) { + c, err := newCipher(NameEncryptionStandard, "", "", true) + assert.NoError(t, err) + + in := &errorReader{io.ErrUnexpectedEOF} + fh, err := c.newEncrypter(in, nil) + assert.NoError(t, err) + + n, err := io.CopyN(ioutil.Discard, fh, 1E6) + assert.Equal(t, io.ErrUnexpectedEOF, err) + assert.Equal(t, int64(32), n) +} + +type errorReader struct { + err error +} + +func (er errorReader) Read(p []byte) (n int, err error) { + return 0, er.err +} + +type closeDetector struct { + io.Reader + closed int +} + +func newCloseDetector(in io.Reader) *closeDetector { + return &closeDetector{ + Reader: in, + } +} + +func (c *closeDetector) Close() error { + c.closed++ + return nil +} + +func TestNewDecrypter(t *testing.T) { + c, err := newCipher(NameEncryptionStandard, "", "", true) + assert.NoError(t, err) + c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator + + cd := newCloseDetector(bytes.NewBuffer(file0)) + fh, err := c.newDecrypter(cd) + assert.NoError(t, err) + // check nonce is in place + assert.Equal(t, file0[8:32], fh.nonce[:]) + assert.Equal(t, 0, cd.closed) + + // Test error paths + for i := range file0 { + cd := newCloseDetector(bytes.NewBuffer(file0[:i])) + fh, err = c.newDecrypter(cd) + assert.Nil(t, fh) + assert.Error(t, err, ErrorEncryptedFileTooShort.Error()) + assert.Equal(t, 1, cd.closed) + } + + er := &errorReader{errors.New("potato")} + cd = newCloseDetector(er) + fh, err = c.newDecrypter(cd) + assert.Nil(t, fh) + assert.Error(t, err, "potato") + assert.Equal(t, 1, cd.closed) + + // bad magic + file0copy := make([]byte, len(file0)) + copy(file0copy, file0) + for i := range fileMagic { + file0copy[i] ^= 0x1 + cd := newCloseDetector(bytes.NewBuffer(file0copy)) + fh, err := c.newDecrypter(cd) + assert.Nil(t, fh) + assert.Error(t, err, ErrorEncryptedBadMagic.Error()) + file0copy[i] ^= 0x1 + assert.Equal(t, 1, cd.closed) + } +} + +// Test the stream returning 0, io.ErrUnexpectedEOF +func TestNewDecrypterErrUnexpectedEOF(t *testing.T) { + c, err := newCipher(NameEncryptionStandard, "", "", true) + assert.NoError(t, err) + + in2 := &errorReader{io.ErrUnexpectedEOF} + in1 := bytes.NewBuffer(file16) + in := ioutil.NopCloser(io.MultiReader(in1, in2)) + + fh, err := c.newDecrypter(in) + assert.NoError(t, err) + + n, err := io.CopyN(ioutil.Discard, fh, 1E6) + assert.Equal(t, io.ErrUnexpectedEOF, err) + assert.Equal(t, int64(16), n) +} + +func TestNewDecrypterSeekLimit(t *testing.T) { + c, err := newCipher(NameEncryptionStandard, "", "", true) + assert.NoError(t, err) + c.cryptoRand = &zeroes{} // nodge the crypto rand generator + + // Make random data + const dataSize = 150000 + plaintext, err := ioutil.ReadAll(newRandomSource(dataSize)) + assert.NoError(t, err) + + // Encrypt the data + buf := bytes.NewBuffer(plaintext) + encrypted, err := c.EncryptData(buf) + assert.NoError(t, err) + ciphertext, err := ioutil.ReadAll(encrypted) + assert.NoError(t, err) + + trials := []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, + 127, 128, 129, 255, 256, 257, 511, 512, 513, 1023, 1024, 1025, 2047, 2048, 2049, + 4095, 4096, 4097, 8191, 8192, 8193, 16383, 16384, 16385, 32767, 32768, 32769, + 65535, 65536, 65537, 131071, 131072, 131073, dataSize - 1, dataSize} + limits := []int{-1, 0, 1, 65535, 65536, 65537, 131071, 131072, 131073} + + // Open stream with a seek of underlyingOffset + var reader io.ReadCloser + open := func(underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) { + end := len(ciphertext) + if underlyingLimit >= 0 { + end = int(underlyingOffset + underlyingLimit) + if end > len(ciphertext) { + end = len(ciphertext) + } + } + reader = ioutil.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end])) + return reader, nil + } + + inBlock := make([]byte, dataSize) + + // Check the seek worked by reading a block and checking it + // against what it should be + check := func(rc io.Reader, offset, limit int) { + n, err := io.ReadFull(rc, inBlock) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + require.NoError(t, err) + } + seekedDecrypted := inBlock[:n] + + what := fmt.Sprintf("offset = %d, limit = %d", offset, limit) + if limit >= 0 { + assert.Equal(t, limit, n, what) + } + require.Equal(t, plaintext[offset:offset+n], seekedDecrypted, what) + + // We should have completely emptied the reader at this point + n, err = reader.Read(inBlock) + assert.Equal(t, io.EOF, err) + assert.Equal(t, 0, n) + } + + // Now try decoding it with a open/seek + for _, offset := range trials { + for _, limit := range limits { + if offset+limit > len(plaintext) { + continue + } + rc, err := c.DecryptDataSeek(open, int64(offset), int64(limit)) + assert.NoError(t, err) + + check(rc, offset, limit) + } + } + + // Try decoding it with a single open and lots of seeks + fh, err := c.DecryptDataSeek(open, 0, -1) + assert.NoError(t, err) + for _, offset := range trials { + for _, limit := range limits { + if offset+limit > len(plaintext) { + continue + } + _, err := fh.RangeSeek(int64(offset), io.SeekStart, int64(limit)) + assert.NoError(t, err) + + check(fh, offset, limit) + } + } + + // Do some checks on the open callback + for _, test := range []struct { + offset, limit int64 + wantOffset, wantLimit int64 + }{ + // unlimited + {0, -1, int64(fileHeaderSize), -1}, + {1, -1, int64(fileHeaderSize), -1}, + {blockDataSize - 1, -1, int64(fileHeaderSize), -1}, + {blockDataSize, -1, int64(fileHeaderSize) + blockSize, -1}, + {blockDataSize + 1, -1, int64(fileHeaderSize) + blockSize, -1}, + // limit=1 + {0, 1, int64(fileHeaderSize), blockSize}, + {1, 1, int64(fileHeaderSize), blockSize}, + {blockDataSize - 1, 1, int64(fileHeaderSize), blockSize}, + {blockDataSize, 1, int64(fileHeaderSize) + blockSize, blockSize}, + {blockDataSize + 1, 1, int64(fileHeaderSize) + blockSize, blockSize}, + // limit=100 + {0, 100, int64(fileHeaderSize), blockSize}, + {1, 100, int64(fileHeaderSize), blockSize}, + {blockDataSize - 1, 100, int64(fileHeaderSize), 2 * blockSize}, + {blockDataSize, 100, int64(fileHeaderSize) + blockSize, blockSize}, + {blockDataSize + 1, 100, int64(fileHeaderSize) + blockSize, blockSize}, + // limit=blockDataSize-1 + {0, blockDataSize - 1, int64(fileHeaderSize), blockSize}, + {1, blockDataSize - 1, int64(fileHeaderSize), blockSize}, + {blockDataSize - 1, blockDataSize - 1, int64(fileHeaderSize), 2 * blockSize}, + {blockDataSize, blockDataSize - 1, int64(fileHeaderSize) + blockSize, blockSize}, + {blockDataSize + 1, blockDataSize - 1, int64(fileHeaderSize) + blockSize, blockSize}, + // limit=blockDataSize + {0, blockDataSize, int64(fileHeaderSize), blockSize}, + {1, blockDataSize, int64(fileHeaderSize), 2 * blockSize}, + {blockDataSize - 1, blockDataSize, int64(fileHeaderSize), 2 * blockSize}, + {blockDataSize, blockDataSize, int64(fileHeaderSize) + blockSize, blockSize}, + {blockDataSize + 1, blockDataSize, int64(fileHeaderSize) + blockSize, 2 * blockSize}, + // limit=blockDataSize+1 + {0, blockDataSize + 1, int64(fileHeaderSize), 2 * blockSize}, + {1, blockDataSize + 1, int64(fileHeaderSize), 2 * blockSize}, + {blockDataSize - 1, blockDataSize + 1, int64(fileHeaderSize), 2 * blockSize}, + {blockDataSize, blockDataSize + 1, int64(fileHeaderSize) + blockSize, 2 * blockSize}, + {blockDataSize + 1, blockDataSize + 1, int64(fileHeaderSize) + blockSize, 2 * blockSize}, + } { + what := fmt.Sprintf("offset = %d, limit = %d", test.offset, test.limit) + callCount := 0 + testOpen := func(underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) { + switch callCount { + case 0: + assert.Equal(t, int64(0), underlyingOffset, what) + assert.Equal(t, int64(-1), underlyingLimit, what) + case 1: + assert.Equal(t, test.wantOffset, underlyingOffset, what) + assert.Equal(t, test.wantLimit, underlyingLimit, what) + default: + t.Errorf("Too many calls %d for %s", callCount+1, what) + } + callCount++ + return open(underlyingOffset, underlyingLimit) + } + fh, err := c.DecryptDataSeek(testOpen, 0, -1) + assert.NoError(t, err) + gotOffset, err := fh.RangeSeek(test.offset, io.SeekStart, test.limit) + assert.NoError(t, err) + assert.Equal(t, gotOffset, test.offset) + } +} + +func TestDecrypterCalculateUnderlying(t *testing.T) { + for _, test := range []struct { + offset, limit int64 + wantOffset, wantLimit int64 + wantDiscard, wantBlocks int64 + }{ + // unlimited + {0, -1, int64(fileHeaderSize), -1, 0, 0}, + {1, -1, int64(fileHeaderSize), -1, 1, 0}, + {blockDataSize - 1, -1, int64(fileHeaderSize), -1, blockDataSize - 1, 0}, + {blockDataSize, -1, int64(fileHeaderSize) + blockSize, -1, 0, 1}, + {blockDataSize + 1, -1, int64(fileHeaderSize) + blockSize, -1, 1, 1}, + // limit=1 + {0, 1, int64(fileHeaderSize), blockSize, 0, 0}, + {1, 1, int64(fileHeaderSize), blockSize, 1, 0}, + {blockDataSize - 1, 1, int64(fileHeaderSize), blockSize, blockDataSize - 1, 0}, + {blockDataSize, 1, int64(fileHeaderSize) + blockSize, blockSize, 0, 1}, + {blockDataSize + 1, 1, int64(fileHeaderSize) + blockSize, blockSize, 1, 1}, + // limit=100 + {0, 100, int64(fileHeaderSize), blockSize, 0, 0}, + {1, 100, int64(fileHeaderSize), blockSize, 1, 0}, + {blockDataSize - 1, 100, int64(fileHeaderSize), 2 * blockSize, blockDataSize - 1, 0}, + {blockDataSize, 100, int64(fileHeaderSize) + blockSize, blockSize, 0, 1}, + {blockDataSize + 1, 100, int64(fileHeaderSize) + blockSize, blockSize, 1, 1}, + // limit=blockDataSize-1 + {0, blockDataSize - 1, int64(fileHeaderSize), blockSize, 0, 0}, + {1, blockDataSize - 1, int64(fileHeaderSize), blockSize, 1, 0}, + {blockDataSize - 1, blockDataSize - 1, int64(fileHeaderSize), 2 * blockSize, blockDataSize - 1, 0}, + {blockDataSize, blockDataSize - 1, int64(fileHeaderSize) + blockSize, blockSize, 0, 1}, + {blockDataSize + 1, blockDataSize - 1, int64(fileHeaderSize) + blockSize, blockSize, 1, 1}, + // limit=blockDataSize + {0, blockDataSize, int64(fileHeaderSize), blockSize, 0, 0}, + {1, blockDataSize, int64(fileHeaderSize), 2 * blockSize, 1, 0}, + {blockDataSize - 1, blockDataSize, int64(fileHeaderSize), 2 * blockSize, blockDataSize - 1, 0}, + {blockDataSize, blockDataSize, int64(fileHeaderSize) + blockSize, blockSize, 0, 1}, + {blockDataSize + 1, blockDataSize, int64(fileHeaderSize) + blockSize, 2 * blockSize, 1, 1}, + // limit=blockDataSize+1 + {0, blockDataSize + 1, int64(fileHeaderSize), 2 * blockSize, 0, 0}, + {1, blockDataSize + 1, int64(fileHeaderSize), 2 * blockSize, 1, 0}, + {blockDataSize - 1, blockDataSize + 1, int64(fileHeaderSize), 2 * blockSize, blockDataSize - 1, 0}, + {blockDataSize, blockDataSize + 1, int64(fileHeaderSize) + blockSize, 2 * blockSize, 0, 1}, + {blockDataSize + 1, blockDataSize + 1, int64(fileHeaderSize) + blockSize, 2 * blockSize, 1, 1}, + } { + what := fmt.Sprintf("offset = %d, limit = %d", test.offset, test.limit) + underlyingOffset, underlyingLimit, discard, blocks := calculateUnderlying(test.offset, test.limit) + assert.Equal(t, test.wantOffset, underlyingOffset, what) + assert.Equal(t, test.wantLimit, underlyingLimit, what) + assert.Equal(t, test.wantDiscard, discard, what) + assert.Equal(t, test.wantBlocks, blocks, what) + } +} + +func TestDecrypterRead(t *testing.T) { + c, err := newCipher(NameEncryptionStandard, "", "", true) + assert.NoError(t, err) + + // Test truncating the file at each possible point + for i := 0; i < len(file16)-1; i++ { + what := fmt.Sprintf("truncating to %d/%d", i, len(file16)) + cd := newCloseDetector(bytes.NewBuffer(file16[:i])) + fh, err := c.newDecrypter(cd) + if i < fileHeaderSize { + assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error(), what) + continue + } + if err != nil { + assert.NoError(t, err, what) + continue + } + _, err = ioutil.ReadAll(fh) + var expectedErr error + switch { + case i == fileHeaderSize: + // This would normally produce an error *except* on the first block + expectedErr = nil + default: + expectedErr = io.ErrUnexpectedEOF + } + if expectedErr != nil { + assert.EqualError(t, err, expectedErr.Error(), what) + } else { + assert.NoError(t, err, what) + } + assert.Equal(t, 0, cd.closed, what) + } + + // Test producing an error on the file on Read the underlying file + in1 := bytes.NewBuffer(file1) + in2 := &errorReader{errors.New("potato")} + in := io.MultiReader(in1, in2) + cd := newCloseDetector(in) + fh, err := c.newDecrypter(cd) + assert.NoError(t, err) + _, err = ioutil.ReadAll(fh) + assert.Error(t, err, "potato") + assert.Equal(t, 0, cd.closed) + + // Test corrupting the input + // shouldn't be able to corrupt any byte without some sort of error + file16copy := make([]byte, len(file16)) + copy(file16copy, file16) + for i := range file16copy { + file16copy[i] ^= 0xFF + fh, err := c.newDecrypter(ioutil.NopCloser(bytes.NewBuffer(file16copy))) + if i < fileMagicSize { + assert.Error(t, err, ErrorEncryptedBadMagic.Error()) + assert.Nil(t, fh) + } else { + assert.NoError(t, err) + _, err = ioutil.ReadAll(fh) + assert.Error(t, err, ErrorEncryptedFileBadHeader.Error()) + } + file16copy[i] ^= 0xFF + } +} + +func TestDecrypterClose(t *testing.T) { + c, err := newCipher(NameEncryptionStandard, "", "", true) + assert.NoError(t, err) + + cd := newCloseDetector(bytes.NewBuffer(file16)) + fh, err := c.newDecrypter(cd) + assert.NoError(t, err) + assert.Equal(t, 0, cd.closed) + + // close before reading + assert.Equal(t, nil, fh.err) + err = fh.Close() + assert.NoError(t, err) + assert.Equal(t, ErrorFileClosed, fh.err) + assert.Equal(t, 1, cd.closed) + + // double close + err = fh.Close() + assert.Error(t, err, ErrorFileClosed.Error()) + assert.Equal(t, 1, cd.closed) + + // try again reading the file this time + cd = newCloseDetector(bytes.NewBuffer(file1)) + fh, err = c.newDecrypter(cd) + assert.NoError(t, err) + assert.Equal(t, 0, cd.closed) + + // close after reading + out, err := ioutil.ReadAll(fh) + assert.NoError(t, err) + assert.Equal(t, []byte{1}, out) + assert.Equal(t, io.EOF, fh.err) + err = fh.Close() + assert.NoError(t, err) + assert.Equal(t, ErrorFileClosed, fh.err) + assert.Equal(t, 1, cd.closed) +} + +func TestPutGetBlock(t *testing.T) { + c, err := newCipher(NameEncryptionStandard, "", "", true) + assert.NoError(t, err) + + block := c.getBlock() + c.putBlock(block) + c.putBlock(block) + + assert.Panics(t, func() { c.putBlock(block[:len(block)-1]) }) +} + +func TestKey(t *testing.T) { + c, err := newCipher(NameEncryptionStandard, "", "", true) + assert.NoError(t, err) + + // Check zero keys OK + assert.Equal(t, [32]byte{}, c.dataKey) + assert.Equal(t, [32]byte{}, c.nameKey) + assert.Equal(t, [16]byte{}, c.nameTweak) + + require.NoError(t, c.Key("potato", "")) + assert.Equal(t, [32]byte{0x74, 0x55, 0xC7, 0x1A, 0xB1, 0x7C, 0x86, 0x5B, 0x84, 0x71, 0xF4, 0x7B, 0x79, 0xAC, 0xB0, 0x7E, 0xB3, 0x1D, 0x56, 0x78, 0xB8, 0x0C, 0x7E, 0x2E, 0xAF, 0x4F, 0xC8, 0x06, 0x6A, 0x9E, 0xE4, 0x68}, c.dataKey) + assert.Equal(t, [32]byte{0x76, 0x5D, 0xA2, 0x7A, 0xB1, 0x5D, 0x77, 0xF9, 0x57, 0x96, 0x71, 0x1F, 0x7B, 0x93, 0xAD, 0x63, 0xBB, 0xB4, 0x84, 0x07, 0x2E, 0x71, 0x80, 0xA8, 0xD1, 0x7A, 0x9B, 0xBE, 0xC1, 0x42, 0x70, 0xD0}, c.nameKey) + assert.Equal(t, [16]byte{0xC1, 0x8D, 0x59, 0x32, 0xF5, 0x5B, 0x28, 0x28, 0xC5, 0xE1, 0xE8, 0x72, 0x15, 0x52, 0x03, 0x10}, c.nameTweak) + + require.NoError(t, c.Key("Potato", "")) + assert.Equal(t, [32]byte{0xAE, 0xEA, 0x6A, 0xD3, 0x47, 0xDF, 0x75, 0xB9, 0x63, 0xCE, 0x12, 0xF5, 0x76, 0x23, 0xE9, 0x46, 0xD4, 0x2E, 0xD8, 0xBF, 0x3E, 0x92, 0x8B, 0x39, 0x24, 0x37, 0x94, 0x13, 0x3E, 0x5E, 0xF7, 0x5E}, c.dataKey) + assert.Equal(t, [32]byte{0x54, 0xF7, 0x02, 0x6E, 0x8A, 0xFC, 0x56, 0x0A, 0x86, 0x63, 0x6A, 0xAB, 0x2C, 0x9C, 0x51, 0x62, 0xE5, 0x1A, 0x12, 0x23, 0x51, 0x83, 0x6E, 0xAF, 0x50, 0x42, 0x0F, 0x98, 0x1C, 0x86, 0x0A, 0x19}, c.nameKey) + assert.Equal(t, [16]byte{0xF8, 0xC1, 0xB6, 0x27, 0x2D, 0x52, 0x9B, 0x4A, 0x8F, 0xDA, 0xEB, 0x42, 0x4A, 0x28, 0xDD, 0xF3}, c.nameTweak) + + require.NoError(t, c.Key("potato", "sausage")) + assert.Equal(t, [32]uint8{0x8e, 0x9b, 0x6b, 0x99, 0xf8, 0x69, 0x4, 0x67, 0xa0, 0x71, 0xf9, 0xcb, 0x92, 0xd0, 0xaa, 0x78, 0x7f, 0x8f, 0xf1, 0x78, 0xbe, 0xc9, 0x6f, 0x99, 0x9f, 0xd5, 0x20, 0x6e, 0x64, 0x4a, 0x1b, 0x50}, c.dataKey) + assert.Equal(t, [32]uint8{0x3e, 0xa9, 0x5e, 0xf6, 0x81, 0x78, 0x2d, 0xc9, 0xd9, 0x95, 0x5d, 0x22, 0x5b, 0xfd, 0x44, 0x2c, 0x6f, 0x5d, 0x68, 0x97, 0xb0, 0x29, 0x1, 0x5c, 0x6f, 0x46, 0x2e, 0x2a, 0x9d, 0xae, 0x2c, 0xe3}, c.nameKey) + assert.Equal(t, [16]uint8{0xf1, 0x7f, 0xd7, 0x14, 0x1d, 0x65, 0x27, 0x4f, 0x36, 0x3f, 0xc2, 0xa0, 0x4d, 0xd2, 0x14, 0x8a}, c.nameTweak) + + require.NoError(t, c.Key("potato", "Sausage")) + assert.Equal(t, [32]uint8{0xda, 0x81, 0x8c, 0x67, 0xef, 0x11, 0xf, 0xc8, 0xd5, 0xc8, 0x62, 0x4b, 0x7f, 0xe2, 0x9e, 0x35, 0x35, 0xb0, 0x8d, 0x79, 0x84, 0x89, 0xac, 0xcb, 0xa0, 0xff, 0x2, 0x72, 0x3, 0x1a, 0x5e, 0x64}, c.dataKey) + assert.Equal(t, [32]uint8{0x2, 0x81, 0x7e, 0x7b, 0xea, 0x99, 0x81, 0x5a, 0xd0, 0x2d, 0xb9, 0x64, 0x48, 0xb0, 0x28, 0x27, 0x7c, 0x20, 0xb4, 0xd4, 0xa4, 0x68, 0xad, 0x4e, 0x5c, 0x29, 0xf, 0x79, 0xef, 0xee, 0xdb, 0x3b}, c.nameKey) + assert.Equal(t, [16]uint8{0x9a, 0xb5, 0xb, 0x3d, 0xcb, 0x60, 0x59, 0x55, 0xa5, 0x4d, 0xe6, 0xb6, 0x47, 0x3, 0x23, 0xe2}, c.nameTweak) + + require.NoError(t, c.Key("", "")) + assert.Equal(t, [32]byte{}, c.dataKey) + assert.Equal(t, [32]byte{}, c.nameKey) + assert.Equal(t, [16]byte{}, c.nameTweak) +} diff --git a/.rclone_repo/backend/crypt/crypt.go b/.rclone_repo/backend/crypt/crypt.go new file mode 100755 index 0000000..a5c6ccc --- /dev/null +++ b/.rclone_repo/backend/crypt/crypt.go @@ -0,0 +1,781 @@ +// Package crypt provides wrappers for Fs and Object which implement encryption +package crypt + +import ( + "fmt" + "io" + "path" + "strings" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/accounting" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/hash" + "github.com/pkg/errors" +) + +// Globals +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "crypt", + Description: "Encrypt/Decrypt a remote", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "remote", + Help: "Remote to encrypt/decrypt.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).", + Required: true, + }, { + Name: "filename_encryption", + Help: "How to encrypt the filenames.", + Default: "standard", + Examples: []fs.OptionExample{ + { + Value: "off", + Help: "Don't encrypt the file names. Adds a \".bin\" extension only.", + }, { + Value: "standard", + Help: "Encrypt the filenames see the docs for the details.", + }, { + Value: "obfuscate", + Help: "Very simple filename obfuscation.", + }, + }, + }, { + Name: "directory_name_encryption", + Help: "Option to either encrypt directory names or leave them intact.", + Default: true, + Examples: []fs.OptionExample{ + { + Value: "true", + Help: "Encrypt directory names.", + }, + { + Value: "false", + Help: "Don't encrypt directory names, leave them intact.", + }, + }, + }, { + Name: "password", + Help: "Password or pass phrase for encryption.", + IsPassword: true, + }, { + Name: "password2", + Help: "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.", + IsPassword: true, + }, { + Name: "show_mapping", + Help: "For all files listed show how the names encrypt.", + Default: false, + Hide: fs.OptionHideConfigurator, + Advanced: true, + }}, + }) +} + +// newCipherForConfig constructs a Cipher for the given config name +func newCipherForConfig(opt *Options) (Cipher, error) { + mode, err := NewNameEncryptionMode(opt.FilenameEncryption) + if err != nil { + return nil, err + } + if opt.Password == "" { + return nil, errors.New("password not set in config file") + } + password, err := obscure.Reveal(opt.Password) + if err != nil { + return nil, errors.Wrap(err, "failed to decrypt password") + } + var salt string + if opt.Password2 != "" { + salt, err = obscure.Reveal(opt.Password2) + if err != nil { + return nil, errors.Wrap(err, "failed to decrypt password2") + } + } + cipher, err := newCipher(mode, password, salt, opt.DirectoryNameEncryption) + if err != nil { + return nil, errors.Wrap(err, "failed to make cipher") + } + return cipher, nil +} + +// NewCipher constructs a Cipher for the given config +func NewCipher(m configmap.Mapper) (Cipher, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + return newCipherForConfig(opt) +} + +// NewFs contstructs an Fs from the path, container:path +func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + cipher, err := newCipherForConfig(opt) + if err != nil { + return nil, err + } + remote := opt.Remote + if strings.HasPrefix(remote, name+":") { + return nil, errors.New("can't point crypt remote at itself - check the value of the remote setting") + } + // Look for a file first + remotePath := path.Join(remote, cipher.EncryptFileName(rpath)) + wrappedFs, err := fs.NewFs(remotePath) + // if that didn't produce a file, look for a directory + if err != fs.ErrorIsFile { + remotePath = path.Join(remote, cipher.EncryptDirName(rpath)) + wrappedFs, err = fs.NewFs(remotePath) + } + if err != fs.ErrorIsFile && err != nil { + return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remotePath) + } + f := &Fs{ + Fs: wrappedFs, + name: name, + root: rpath, + opt: *opt, + cipher: cipher, + } + // the features here are ones we could support, and they are + // ANDed with the ones from wrappedFs + f.features = (&fs.Features{ + CaseInsensitive: cipher.NameEncryptionMode() == NameEncryptionOff, + DuplicateFiles: true, + ReadMimeType: false, // MimeTypes not supported with crypt + WriteMimeType: false, + BucketBased: true, + CanHaveEmptyDirectories: true, + }).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs) + + doChangeNotify := wrappedFs.Features().ChangeNotify + if doChangeNotify != nil { + f.features.ChangeNotify = func(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool { + wrappedNotifyFunc := func(path string, entryType fs.EntryType) { + decrypted, err := f.DecryptFileName(path) + if err != nil { + fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err) + return + } + notifyFunc(decrypted, entryType) + } + return doChangeNotify(wrappedNotifyFunc, pollInterval) + } + } + + return f, err +} + +// Options defines the configuration for this backend +type Options struct { + Remote string `config:"remote"` + FilenameEncryption string `config:"filename_encryption"` + DirectoryNameEncryption bool `config:"directory_name_encryption"` + Password string `config:"password"` + Password2 string `config:"password2"` + ShowMapping bool `config:"show_mapping"` +} + +// Fs represents a wrapped fs.Fs +type Fs struct { + fs.Fs + name string + root string + opt Options + features *fs.Features // optional features + cipher Cipher +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// String returns a description of the FS +func (f *Fs) String() string { + return fmt.Sprintf("Encrypted drive '%s:%s'", f.name, f.root) +} + +// Encrypt an object file name to entries. +func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) { + remote := obj.Remote() + decryptedRemote, err := f.cipher.DecryptFileName(remote) + if err != nil { + fs.Debugf(remote, "Skipping undecryptable file name: %v", err) + return + } + if f.opt.ShowMapping { + fs.Logf(decryptedRemote, "Encrypts to %q", remote) + } + *entries = append(*entries, f.newObject(obj)) +} + +// Encrypt an directory file name to entries. +func (f *Fs) addDir(entries *fs.DirEntries, dir fs.Directory) { + remote := dir.Remote() + decryptedRemote, err := f.cipher.DecryptDirName(remote) + if err != nil { + fs.Debugf(remote, "Skipping undecryptable dir name: %v", err) + return + } + if f.opt.ShowMapping { + fs.Logf(decryptedRemote, "Encrypts to %q", remote) + } + *entries = append(*entries, f.newDir(dir)) +} + +// Encrypt some directory entries. This alters entries returning it as newEntries. +func (f *Fs) encryptEntries(entries fs.DirEntries) (newEntries fs.DirEntries, err error) { + newEntries = entries[:0] // in place filter + for _, entry := range entries { + switch x := entry.(type) { + case fs.Object: + f.add(&newEntries, x) + case fs.Directory: + f.addDir(&newEntries, x) + default: + return nil, errors.Errorf("Unknown object type %T", entry) + } + } + return newEntries, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + entries, err = f.Fs.List(f.cipher.EncryptDirName(dir)) + if err != nil { + return nil, err + } + return f.encryptEntries(entries) +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + return f.Fs.Features().ListR(f.cipher.EncryptDirName(dir), func(entries fs.DirEntries) error { + newEntries, err := f.encryptEntries(entries) + if err != nil { + return err + } + return callback(newEntries) + }) +} + +// NewObject finds the Object at remote. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + o, err := f.Fs.NewObject(f.cipher.EncryptFileName(remote)) + if err != nil { + return nil, err + } + return f.newObject(o), nil +} + +type putFn func(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) + +// put implements Put or PutStream +func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) { + // Encrypt the data into wrappedIn + wrappedIn, err := f.cipher.EncryptData(in) + if err != nil { + return nil, err + } + + // Find a hash the destination supports to compute a hash of + // the encrypted data + ht := f.Fs.Hashes().GetOne() + var hasher *hash.MultiHasher + if ht != hash.None { + hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht)) + if err != nil { + return nil, err + } + // unwrap the accounting + var wrap accounting.WrapFn + wrappedIn, wrap = accounting.UnWrap(wrappedIn) + // add the hasher + wrappedIn = io.TeeReader(wrappedIn, hasher) + // wrap the accounting back on + wrappedIn = wrap(wrappedIn) + } + + // Transfer the data + o, err := put(wrappedIn, f.newObjectInfo(src), options...) + if err != nil { + return nil, err + } + + // Check the hashes of the encrypted data if we were comparing them + if ht != hash.None && hasher != nil { + srcHash := hasher.Sums()[ht] + var dstHash string + dstHash, err = o.Hash(ht) + if err != nil { + return nil, errors.Wrap(err, "failed to read destination hash") + } + if srcHash != "" && dstHash != "" && srcHash != dstHash { + // remove object + err = o.Remove() + if err != nil { + fs.Errorf(o, "Failed to remove corrupted object: %v", err) + } + return nil, errors.Errorf("corrupted on transfer: %v crypted hash differ %q vs %q", ht, srcHash, dstHash) + } + } + + return f.newObject(o), nil +} + +// Put in to the remote path with the modTime given of the given size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.put(in, src, options, f.Fs.Put) +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.put(in, src, options, f.Fs.Features().PutStream) +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.None) +} + +// Mkdir makes the directory (container, bucket) +// +// Shouldn't return an error if it already exists +func (f *Fs) Mkdir(dir string) error { + return f.Fs.Mkdir(f.cipher.EncryptDirName(dir)) +} + +// Rmdir removes the directory (container, bucket) if empty +// +// Return an error if it doesn't exist or isn't empty +func (f *Fs) Rmdir(dir string) error { + return f.Fs.Rmdir(f.cipher.EncryptDirName(dir)) +} + +// Purge all files in the root and the root directory +// +// Implement this if you have a way of deleting all the files +// quicker than just running Remove() on the result of List() +// +// Return an error if it doesn't exist +func (f *Fs) Purge() error { + do := f.Fs.Features().Purge + if do == nil { + return fs.ErrorCantPurge + } + return do() +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + do := f.Fs.Features().Copy + if do == nil { + return nil, fs.ErrorCantCopy + } + o, ok := src.(*Object) + if !ok { + return nil, fs.ErrorCantCopy + } + oResult, err := do(o.Object, f.cipher.EncryptFileName(remote)) + if err != nil { + return nil, err + } + return f.newObject(oResult), nil +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + do := f.Fs.Features().Move + if do == nil { + return nil, fs.ErrorCantMove + } + o, ok := src.(*Object) + if !ok { + return nil, fs.ErrorCantMove + } + oResult, err := do(o.Object, f.cipher.EncryptFileName(remote)) + if err != nil { + return nil, err + } + return f.newObject(oResult), nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + do := f.Fs.Features().DirMove + if do == nil { + return fs.ErrorCantDirMove + } + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + return do(srcFs.Fs, f.cipher.EncryptDirName(srcRemote), f.cipher.EncryptDirName(dstRemote)) +} + +// PutUnchecked uploads the object +// +// This will create a duplicate if we upload a new file without +// checking to see if there is one already - use Put() for that. +func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + do := f.Fs.Features().PutUnchecked + if do == nil { + return nil, errors.New("can't PutUnchecked") + } + wrappedIn, err := f.cipher.EncryptData(in) + if err != nil { + return nil, err + } + o, err := do(wrappedIn, f.newObjectInfo(src)) + if err != nil { + return nil, err + } + return f.newObject(o), nil +} + +// CleanUp the trash in the Fs +// +// Implement this if you have a way of emptying the trash or +// otherwise cleaning up old versions of files. +func (f *Fs) CleanUp() error { + do := f.Fs.Features().CleanUp + if do == nil { + return errors.New("can't CleanUp") + } + return do() +} + +// About gets quota information from the Fs +func (f *Fs) About() (*fs.Usage, error) { + do := f.Fs.Features().About + if do == nil { + return nil, errors.New("About not supported") + } + return do() +} + +// UnWrap returns the Fs that this Fs is wrapping +func (f *Fs) UnWrap() fs.Fs { + return f.Fs +} + +// EncryptFileName returns an encrypted file name +func (f *Fs) EncryptFileName(fileName string) string { + return f.cipher.EncryptFileName(fileName) +} + +// DecryptFileName returns a decrypted file name +func (f *Fs) DecryptFileName(encryptedFileName string) (string, error) { + return f.cipher.DecryptFileName(encryptedFileName) +} + +// ComputeHash takes the nonce from o, and encrypts the contents of +// src with it, and calcuates the hash given by HashType on the fly +// +// Note that we break lots of encapsulation in this function. +func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) { + // Read the nonce - opening the file is sufficient to read the nonce in + // use a limited read so we only read the header + in, err := o.Object.Open(&fs.RangeOption{Start: 0, End: int64(fileHeaderSize) - 1}) + if err != nil { + return "", errors.Wrap(err, "failed to open object to read nonce") + } + d, err := f.cipher.(*cipher).newDecrypter(in) + if err != nil { + _ = in.Close() + return "", errors.Wrap(err, "failed to open object to read nonce") + } + nonce := d.nonce + // fs.Debugf(o, "Read nonce % 2x", nonce) + + // Check nonce isn't all zeros + isZero := true + for i := range nonce { + if nonce[i] != 0 { + isZero = false + } + } + if isZero { + fs.Errorf(o, "empty nonce read") + } + + // Close d (and hence in) once we have read the nonce + err = d.Close() + if err != nil { + return "", errors.Wrap(err, "failed to close nonce read") + } + + // Open the src for input + in, err = src.Open() + if err != nil { + return "", errors.Wrap(err, "failed to open src") + } + defer fs.CheckClose(in, &err) + + // Now encrypt the src with the nonce + out, err := f.cipher.(*cipher).newEncrypter(in, &nonce) + if err != nil { + return "", errors.Wrap(err, "failed to make encrypter") + } + + // pipe into hash + m, err := hash.NewMultiHasherTypes(hash.NewHashSet(hashType)) + if err != nil { + return "", errors.Wrap(err, "failed to make hasher") + } + _, err = io.Copy(m, out) + if err != nil { + return "", errors.Wrap(err, "failed to hash data") + } + + return m.Sums()[hashType], nil +} + +// Object describes a wrapped for being read from the Fs +// +// This decrypts the remote name and decrypts the data +type Object struct { + fs.Object + f *Fs +} + +func (f *Fs) newObject(o fs.Object) *Object { + return &Object{ + Object: o, + f: f, + } +} + +// Fs returns read only access to the Fs that this object is part of +func (o *Object) Fs() fs.Info { + return o.f +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.Remote() +} + +// Remote returns the remote path +func (o *Object) Remote() string { + remote := o.Object.Remote() + decryptedName, err := o.f.cipher.DecryptFileName(remote) + if err != nil { + fs.Debugf(remote, "Undecryptable file name: %v", err) + return remote + } + return decryptedName +} + +// Size returns the size of the file +func (o *Object) Size() int64 { + size, err := o.f.cipher.DecryptedSize(o.Object.Size()) + if err != nil { + fs.Debugf(o, "Bad size for decrypt: %v", err) + } + return size +} + +// Hash returns the selected checksum of the file +// If no checksum is available it returns "" +func (o *Object) Hash(ht hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// UnWrap returns the wrapped Object +func (o *Object) UnWrap() fs.Object { + return o.Object +} + +// Open opens the file for read. Call Close() on the returned io.ReadCloser +func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) { + var openOptions []fs.OpenOption + var offset, limit int64 = 0, -1 + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + case *fs.RangeOption: + offset, limit = x.Decode(o.Size()) + default: + // pass on Options to underlying open if appropriate + openOptions = append(openOptions, option) + } + } + rc, err = o.f.cipher.DecryptDataSeek(func(underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) { + if underlyingOffset == 0 && underlyingLimit < 0 { + // Open with no seek + return o.Object.Open(openOptions...) + } + // Open stream with a range of underlyingOffset, underlyingLimit + end := int64(-1) + if underlyingLimit >= 0 { + end = underlyingOffset + underlyingLimit - 1 + if end >= o.Object.Size() { + end = -1 + } + } + newOpenOptions := append(openOptions, &fs.RangeOption{Start: underlyingOffset, End: end}) + return o.Object.Open(newOpenOptions...) + }, offset, limit) + if err != nil { + return nil, err + } + return rc, nil +} + +// Update in to the object with the modTime given of the given size +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + update := func(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return o.Object, o.Object.Update(in, src, options...) + } + _, err := o.f.put(in, src, options, update) + return err +} + +// newDir returns a dir with the Name decrypted +func (f *Fs) newDir(dir fs.Directory) fs.Directory { + newDir := fs.NewDirCopy(dir) + remote := dir.Remote() + decryptedRemote, err := f.cipher.DecryptDirName(remote) + if err != nil { + fs.Debugf(remote, "Undecryptable dir name: %v", err) + } else { + newDir.SetRemote(decryptedRemote) + } + return newDir +} + +// ObjectInfo describes a wrapped fs.ObjectInfo for being the source +// +// This encrypts the remote name and adjusts the size +type ObjectInfo struct { + fs.ObjectInfo + f *Fs +} + +func (f *Fs) newObjectInfo(src fs.ObjectInfo) *ObjectInfo { + return &ObjectInfo{ + ObjectInfo: src, + f: f, + } +} + +// Fs returns read only access to the Fs that this object is part of +func (o *ObjectInfo) Fs() fs.Info { + return o.f +} + +// Remote returns the remote path +func (o *ObjectInfo) Remote() string { + return o.f.cipher.EncryptFileName(o.ObjectInfo.Remote()) +} + +// Size returns the size of the file +func (o *ObjectInfo) Size() int64 { + size := o.ObjectInfo.Size() + if size < 0 { + return size + } + return o.f.cipher.EncryptedSize(size) +} + +// Hash returns the selected checksum of the file +// If no checksum is available it returns "" +func (o *ObjectInfo) Hash(hash hash.Type) (string, error) { + return "", nil +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.PutUncheckeder = (*Fs)(nil) + _ fs.PutStreamer = (*Fs)(nil) + _ fs.CleanUpper = (*Fs)(nil) + _ fs.UnWrapper = (*Fs)(nil) + _ fs.ListRer = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) + _ fs.ObjectInfo = (*ObjectInfo)(nil) + _ fs.Object = (*Object)(nil) + _ fs.ObjectUnWrapper = (*Object)(nil) +) diff --git a/.rclone_repo/backend/crypt/crypt_test.go b/.rclone_repo/backend/crypt/crypt_test.go new file mode 100755 index 0000000..1fb14ab --- /dev/null +++ b/.rclone_repo/backend/crypt/crypt_test.go @@ -0,0 +1,62 @@ +// Test Crypt filesystem interface +package crypt_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/ncw/rclone/backend/crypt" + _ "github.com/ncw/rclone/backend/local" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestStandard runs integration tests against the remote +func TestStandard(t *testing.T) { + tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard") + name := "TestCrypt" + fstests.Run(t, &fstests.Opt{ + RemoteName: name + ":", + NilObject: (*crypt.Object)(nil), + ExtraConfig: []fstests.ExtraConfigItem{ + {Name: name, Key: "type", Value: "crypt"}, + {Name: name, Key: "remote", Value: tempdir}, + {Name: name, Key: "password", Value: obscure.MustObscure("potato")}, + {Name: name, Key: "filename_encryption", Value: "standard"}, + }, + }) +} + +// TestOff runs integration tests against the remote +func TestOff(t *testing.T) { + tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-off") + name := "TestCrypt2" + fstests.Run(t, &fstests.Opt{ + RemoteName: name + ":", + NilObject: (*crypt.Object)(nil), + ExtraConfig: []fstests.ExtraConfigItem{ + {Name: name, Key: "type", Value: "crypt"}, + {Name: name, Key: "remote", Value: tempdir}, + {Name: name, Key: "password", Value: obscure.MustObscure("potato2")}, + {Name: name, Key: "filename_encryption", Value: "off"}, + }, + }) +} + +// TestObfuscate runs integration tests against the remote +func TestObfuscate(t *testing.T) { + tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate") + name := "TestCrypt3" + fstests.Run(t, &fstests.Opt{ + RemoteName: name + ":", + NilObject: (*crypt.Object)(nil), + ExtraConfig: []fstests.ExtraConfigItem{ + {Name: name, Key: "type", Value: "crypt"}, + {Name: name, Key: "remote", Value: tempdir}, + {Name: name, Key: "password", Value: obscure.MustObscure("potato2")}, + {Name: name, Key: "filename_encryption", Value: "obfuscate"}, + }, + SkipBadWindowsCharacters: true, + }) +} diff --git a/.rclone_repo/backend/crypt/pkcs7/pkcs7.go b/.rclone_repo/backend/crypt/pkcs7/pkcs7.go new file mode 100755 index 0000000..e6d9d0f --- /dev/null +++ b/.rclone_repo/backend/crypt/pkcs7/pkcs7.go @@ -0,0 +1,63 @@ +// Package pkcs7 implements PKCS#7 padding +// +// This is a standard way of encoding variable length buffers into +// buffers which are a multiple of an underlying crypto block size. +package pkcs7 + +import "github.com/pkg/errors" + +// Errors Unpad can return +var ( + ErrorPaddingNotFound = errors.New("Bad PKCS#7 padding - not padded") + ErrorPaddingNotAMultiple = errors.New("Bad PKCS#7 padding - not a multiple of blocksize") + ErrorPaddingTooLong = errors.New("Bad PKCS#7 padding - too long") + ErrorPaddingTooShort = errors.New("Bad PKCS#7 padding - too short") + ErrorPaddingNotAllTheSame = errors.New("Bad PKCS#7 padding - not all the same") +) + +// Pad buf using PKCS#7 to a multiple of n. +// +// Appends the padding to buf - make a copy of it first if you don't +// want it modified. +func Pad(n int, buf []byte) []byte { + if n <= 1 || n >= 256 { + panic("bad multiple") + } + length := len(buf) + padding := n - (length % n) + for i := 0; i < padding; i++ { + buf = append(buf, byte(padding)) + } + if (len(buf) % n) != 0 { + panic("padding failed") + } + return buf +} + +// Unpad buf using PKCS#7 from a multiple of n returning a slice of +// buf or an error if malformed. +func Unpad(n int, buf []byte) ([]byte, error) { + if n <= 1 || n >= 256 { + panic("bad multiple") + } + length := len(buf) + if length == 0 { + return nil, ErrorPaddingNotFound + } + if (length % n) != 0 { + return nil, ErrorPaddingNotAMultiple + } + padding := int(buf[length-1]) + if padding > n { + return nil, ErrorPaddingTooLong + } + if padding == 0 { + return nil, ErrorPaddingTooShort + } + for i := 0; i < padding; i++ { + if buf[length-1-i] != byte(padding) { + return nil, ErrorPaddingNotAllTheSame + } + } + return buf[:length-padding], nil +} diff --git a/.rclone_repo/backend/crypt/pkcs7/pkcs7_test.go b/.rclone_repo/backend/crypt/pkcs7/pkcs7_test.go new file mode 100755 index 0000000..2264c7f --- /dev/null +++ b/.rclone_repo/backend/crypt/pkcs7/pkcs7_test.go @@ -0,0 +1,73 @@ +package pkcs7 + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPad(t *testing.T) { + for _, test := range []struct { + n int + in string + expected string + }{ + {8, "", "\x08\x08\x08\x08\x08\x08\x08\x08"}, + {8, "1", "1\x07\x07\x07\x07\x07\x07\x07"}, + {8, "12", "12\x06\x06\x06\x06\x06\x06"}, + {8, "123", "123\x05\x05\x05\x05\x05"}, + {8, "1234", "1234\x04\x04\x04\x04"}, + {8, "12345", "12345\x03\x03\x03"}, + {8, "123456", "123456\x02\x02"}, + {8, "1234567", "1234567\x01"}, + {8, "abcdefgh", "abcdefgh\x08\x08\x08\x08\x08\x08\x08\x08"}, + {8, "abcdefgh1", "abcdefgh1\x07\x07\x07\x07\x07\x07\x07"}, + {8, "abcdefgh12", "abcdefgh12\x06\x06\x06\x06\x06\x06"}, + {8, "abcdefgh123", "abcdefgh123\x05\x05\x05\x05\x05"}, + {8, "abcdefgh1234", "abcdefgh1234\x04\x04\x04\x04"}, + {8, "abcdefgh12345", "abcdefgh12345\x03\x03\x03"}, + {8, "abcdefgh123456", "abcdefgh123456\x02\x02"}, + {8, "abcdefgh1234567", "abcdefgh1234567\x01"}, + {8, "abcdefgh12345678", "abcdefgh12345678\x08\x08\x08\x08\x08\x08\x08\x08"}, + {16, "", "\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10"}, + {16, "a", "a\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f"}, + } { + actual := Pad(test.n, []byte(test.in)) + assert.Equal(t, test.expected, string(actual), fmt.Sprintf("Pad %d %q", test.n, test.in)) + recovered, err := Unpad(test.n, actual) + assert.NoError(t, err) + assert.Equal(t, []byte(test.in), recovered, fmt.Sprintf("Unpad %d %q", test.n, test.in)) + } + assert.Panics(t, func() { Pad(1, []byte("")) }, "bad multiple") + assert.Panics(t, func() { Pad(256, []byte("")) }, "bad multiple") +} + +func TestUnpad(t *testing.T) { + // We've tested the OK decoding in TestPad, now test the error cases + for _, test := range []struct { + n int + in string + err error + }{ + {8, "", ErrorPaddingNotFound}, + {8, "1", ErrorPaddingNotAMultiple}, + {8, "12", ErrorPaddingNotAMultiple}, + {8, "123", ErrorPaddingNotAMultiple}, + {8, "1234", ErrorPaddingNotAMultiple}, + {8, "12345", ErrorPaddingNotAMultiple}, + {8, "123456", ErrorPaddingNotAMultiple}, + {8, "1234567", ErrorPaddingNotAMultiple}, + {8, "1234567\xFF", ErrorPaddingTooLong}, + {8, "1234567\x09", ErrorPaddingTooLong}, + {8, "1234567\x00", ErrorPaddingTooShort}, + {8, "123456\x01\x02", ErrorPaddingNotAllTheSame}, + {8, "\x07\x08\x08\x08\x08\x08\x08\x08", ErrorPaddingNotAllTheSame}, + } { + result, actualErr := Unpad(test.n, []byte(test.in)) + assert.Equal(t, test.err, actualErr, fmt.Sprintf("Unpad %d %q", test.n, test.in)) + assert.Equal(t, result, []byte(nil)) + } + assert.Panics(t, func() { _, _ = Unpad(1, []byte("")) }, "bad multiple") + assert.Panics(t, func() { _, _ = Unpad(256, []byte("")) }, "bad multiple") +} diff --git a/.rclone_repo/backend/drive/drive.go b/.rclone_repo/backend/drive/drive.go new file mode 100755 index 0000000..c818de8 --- /dev/null +++ b/.rclone_repo/backend/drive/drive.go @@ -0,0 +1,2088 @@ +// Package drive interfaces with the Google Drive object storage system +package drive + +// FIXME need to deal with some corner cases +// * multiple files with the same name +// * files can be in multiple directories +// * can have directory loops +// * files with / in name + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "net/url" + "os" + "path" + "strconv" + "strings" + "sync" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/fs/walk" + "github.com/ncw/rclone/lib/dircache" + "github.com/ncw/rclone/lib/oauthutil" + "github.com/ncw/rclone/lib/pacer" + "github.com/pkg/errors" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + drive "google.golang.org/api/drive/v3" + "google.golang.org/api/googleapi" +) + +// Constants +const ( + rcloneClientID = "202264815644.apps.googleusercontent.com" + rcloneEncryptedClientSecret = "eX8GpZTVx3vxMWVkuuBdDWmAUE6rGhTwVrvG9GhllYccSdj2-mvHVg" + driveFolderType = "application/vnd.google-apps.folder" + timeFormatIn = time.RFC3339 + timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00" + minSleep = 10 * time.Millisecond + defaultExtensions = "docx,xlsx,pptx,svg" + scopePrefix = "https://www.googleapis.com/auth/" + defaultScope = "drive" + // chunkSize is the size of the chunks created during a resumable upload and should be a power of two. + // 1<<18 is the minimum size supported by the Google uploader, and there is no maximum. + defaultChunkSize = fs.SizeSuffix(8 * 1024 * 1024) +) + +// Globals +var ( + // Description of how to auth for this app + driveConfig = &oauth2.Config{ + Scopes: []string{scopePrefix + "drive"}, + Endpoint: google.Endpoint, + ClientID: rcloneClientID, + ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), + RedirectURL: oauthutil.TitleBarRedirectURL, + } + mimeTypeToExtension = map[string]string{ + "application/epub+zip": "epub", + "application/msword": "doc", + "application/pdf": "pdf", + "application/rtf": "rtf", + "application/vnd.ms-excel": "xls", + "application/vnd.oasis.opendocument.presentation": "odp", + "application/vnd.oasis.opendocument.spreadsheet": "ods", + "application/vnd.oasis.opendocument.text": "odt", + "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", + "application/x-vnd.oasis.opendocument.spreadsheet": "ods", + "application/zip": "zip", + "image/jpeg": "jpg", + "image/png": "png", + "image/svg+xml": "svg", + "text/csv": "csv", + "text/html": "html", + "text/plain": "txt", + "text/tab-separated-values": "tsv", + } + extensionToMimeType map[string]string + partialFields = "id,name,size,md5Checksum,trashed,modifiedTime,createdTime,mimeType,parents" + exportFormatsOnce sync.Once // make sure we fetch the export formats only once + _exportFormats map[string][]string // allowed export mime-type conversions +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "drive", + Description: "Google Drive", + NewFs: NewFs, + Config: func(name string, m configmap.Mapper) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + fs.Errorf(nil, "Couldn't parse config into struct: %v", err) + return + } + // Fill in the scopes + if opt.Scope == "" { + opt.Scope = defaultScope + } + driveConfig.Scopes = nil + for _, scope := range strings.Split(opt.Scope, ",") { + driveConfig.Scopes = append(driveConfig.Scopes, scopePrefix+strings.TrimSpace(scope)) + // Set the root_folder_id if using drive.appfolder + if scope == "drive.appfolder" { + m.Set("root_folder_id", "appDataFolder") + } + } + if opt.ServiceAccountFile == "" { + err = oauthutil.Config("drive", name, m, driveConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + } + err = configTeamDrive(opt, m, name) + if err != nil { + log.Fatalf("Failed to configure team drive: %v", err) + } + }, + Options: []fs.Option{{ + Name: config.ConfigClientID, + Help: "Google Application Client Id\nLeave blank normally.", + }, { + Name: config.ConfigClientSecret, + Help: "Google Application Client Secret\nLeave blank normally.", + }, { + Name: "scope", + Help: "Scope that rclone should use when requesting access from drive.", + Examples: []fs.OptionExample{{ + Value: "drive", + Help: "Full access all files, excluding Application Data Folder.", + }, { + Value: "drive.readonly", + Help: "Read-only access to file metadata and file contents.", + }, { + Value: "drive.file", + Help: "Access to files created by rclone only.\nThese are visible in the drive website.\nFile authorization is revoked when the user deauthorizes the app.", + }, { + Value: "drive.appfolder", + Help: "Allows read and write access to the Application Data folder.\nThis is not visible in the drive website.", + }, { + Value: "drive.metadata.readonly", + Help: "Allows read-only access to file metadata but\ndoes not allow any access to read or download file content.", + }}, + }, { + Name: "root_folder_id", + Help: "ID of the root folder\nLeave blank normally.\nFill in to access \"Computers\" folders. (see docs).", + }, { + Name: "service_account_file", + Help: "Service Account Credentials JSON file path \nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.", + }, { + Name: "service_account_credentials", + Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.", + Hide: fs.OptionHideBoth, + Advanced: true, + }, { + Name: "team_drive", + Help: "ID of the Team Drive", + Hide: fs.OptionHideBoth, + Advanced: true, + }, { + Name: "auth_owner_only", + Default: false, + Help: "Only consider files owned by the authenticated user.", + Advanced: true, + }, { + Name: "use_trash", + Default: true, + Help: "Send files to the trash instead of deleting permanently.", + Advanced: true, + }, { + Name: "skip_gdocs", + Default: false, + Help: "Skip google documents in all listings.", + Advanced: true, + }, { + Name: "shared_with_me", + Default: false, + Help: "Only show files that are shared with me", + Advanced: true, + }, { + Name: "trashed_only", + Default: false, + Help: "Only show files that are in the trash", + Advanced: true, + }, { + Name: "formats", + Default: defaultExtensions, + Help: "Comma separated list of preferred formats for downloading Google docs.", + Advanced: true, + }, { + Name: "use_created_date", + Default: false, + Help: "Use created date instead of modified date.", + Advanced: true, + }, { + Name: "list_chunk", + Default: 1000, + Help: "Size of listing chunk 100-1000. 0 to disable.", + Advanced: true, + }, { + Name: "impersonate", + Default: "", + Help: "Impersonate this user when using a service account.", + Advanced: true, + }, { + Name: "alternate_export", + Default: false, + Help: "Use alternate export URLs for google documents export.", + Advanced: true, + }, { + Name: "upload_cutoff", + Default: defaultChunkSize, + Help: "Cutoff for switching to chunked upload", + Advanced: true, + }, { + Name: "chunk_size", + Default: defaultChunkSize, + Help: "Upload chunk size. Must a power of 2 >= 256k.", + Advanced: true, + }, { + Name: "acknowledge_abuse", + Default: false, + Help: "Set to allow files which return cannotDownloadAbusiveFile to be downloaded.", + Advanced: true, + }, { + Name: "keep_revision_forever", + Default: false, + Help: "Keep new head revision forever.", + Advanced: true, + }}, + }) + + // Invert mimeTypeToExtension + extensionToMimeType = make(map[string]string, len(mimeTypeToExtension)) + for mimeType, extension := range mimeTypeToExtension { + extensionToMimeType[extension] = mimeType + } +} + +// Options defines the configuration for this backend +type Options struct { + Scope string `config:"scope"` + RootFolderID string `config:"root_folder_id"` + ServiceAccountFile string `config:"service_account_file"` + ServiceAccountCredentials string `config:"service_account_credentials"` + TeamDriveID string `config:"team_drive"` + AuthOwnerOnly bool `config:"auth_owner_only"` + UseTrash bool `config:"use_trash"` + SkipGdocs bool `config:"skip_gdocs"` + SharedWithMe bool `config:"shared_with_me"` + TrashedOnly bool `config:"trashed_only"` + Extensions string `config:"formats"` + UseCreatedDate bool `config:"use_created_date"` + ListChunk int64 `config:"list_chunk"` + Impersonate string `config:"impersonate"` + AlternateExport bool `config:"alternate_export"` + UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` + AcknowledgeAbuse bool `config:"acknowledge_abuse"` + KeepRevisionForever bool `config:"keep_revision_forever"` +} + +// Fs represents a remote drive server +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + features *fs.Features // optional features + svc *drive.Service // the connection to the drive server + client *http.Client // authorized client + rootFolderID string // the id of the root folder + dirCache *dircache.DirCache // Map of directory path to directory id + pacer *pacer.Pacer // To pace the API calls + extensions []string // preferred extensions to download docs + isTeamDrive bool // true if this is a team drive +} + +// Object describes a drive object +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + id string // Drive Id of this object + url string // Download URL of this object + md5sum string // md5sum of the object + bytes int64 // size of the object + modifiedDate string // RFC3339 time it was last modified + isDocument bool // if set this is a Google doc + mimeType string +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("Google drive root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// shouldRetry determines whehter a given err rates being retried +func shouldRetry(err error) (again bool, errOut error) { + again = false + if err != nil { + if fserrors.ShouldRetry(err) { + again = true + } else { + switch gerr := err.(type) { + case *googleapi.Error: + if gerr.Code >= 500 && gerr.Code < 600 { + // All 5xx errors should be retried + again = true + } else if len(gerr.Errors) > 0 { + reason := gerr.Errors[0].Reason + if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" { + again = true + } + } + } + } + } + return again, err +} + +// parseParse parses a drive 'url' +func parseDrivePath(path string) (root string, err error) { + root = strings.Trim(path, "/") + return +} + +// User function to process a File item from list +// +// Should return true to finish processing +type listFn func(*drive.File) bool + +func containsString(slice []string, s string) bool { + for _, e := range slice { + if e == s { + return true + } + } + return false +} + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +// +// Search params: https://developers.google.com/drive/search-parameters +func (f *Fs) list(dirIDs []string, title string, directoriesOnly bool, filesOnly bool, includeAll bool, fn listFn) (found bool, err error) { + var query []string + if !includeAll { + q := "trashed=" + strconv.FormatBool(f.opt.TrashedOnly) + if f.opt.TrashedOnly { + q = fmt.Sprintf("(mimeType='%s' or %s)", driveFolderType, q) + } + query = append(query, q) + } + // Search with sharedWithMe will always return things listed in "Shared With Me" (without any parents) + // We must not filter with parent when we try list "ROOT" with drive-shared-with-me + // If we need to list file inside those shared folders, we must search it without sharedWithMe + parentsQuery := bytes.NewBufferString("(") + for _, dirID := range dirIDs { + if dirID == "" { + continue + } + if parentsQuery.Len() > 1 { + _, _ = parentsQuery.WriteString(" or ") + } + if f.opt.SharedWithMe && dirID == f.rootFolderID { + _, _ = parentsQuery.WriteString("sharedWithMe=true") + } else { + _, _ = fmt.Fprintf(parentsQuery, "'%s' in parents", dirID) + } + } + if parentsQuery.Len() > 1 { + _ = parentsQuery.WriteByte(')') + query = append(query, parentsQuery.String()) + } + stem := "" + if title != "" { + // Escaping the backslash isn't documented but seems to work + searchTitle := strings.Replace(title, `\`, `\\`, -1) + searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1) + // Convert ／ to / for search + searchTitle = strings.Replace(searchTitle, "／", "/", -1) + + handleGdocs := !directoriesOnly && !f.opt.SkipGdocs + // if the search title contains an extension and the extension is in the export extensions add a search + // for the filename without the extension. + // assume that export extensions don't contain escape sequences and only have one part (not .tar.gz) + if ext := path.Ext(searchTitle); handleGdocs && len(ext) > 0 && containsString(f.extensions, ext[1:]) { + stem = title[:len(title)-len(ext)] + query = append(query, fmt.Sprintf("(name='%s' or name='%s')", searchTitle, searchTitle[:len(title)-len(ext)])) + } else { + query = append(query, fmt.Sprintf("name='%s'", searchTitle)) + } + } + if directoriesOnly { + query = append(query, fmt.Sprintf("mimeType='%s'", driveFolderType)) + } + if filesOnly { + query = append(query, fmt.Sprintf("mimeType!='%s'", driveFolderType)) + } + list := f.svc.Files.List() + if len(query) > 0 { + list.Q(strings.Join(query, " and ")) + // fmt.Printf("list Query = %q\n", query) + } + if f.opt.ListChunk > 0 { + list.PageSize(f.opt.ListChunk) + } + if f.isTeamDrive { + list.TeamDriveId(f.opt.TeamDriveID) + list.SupportsTeamDrives(true) + list.IncludeTeamDriveItems(true) + list.Corpora("teamDrive") + } + // If using appDataFolder then need to add Spaces + if f.rootFolderID == "appDataFolder" { + list.Spaces("appDataFolder") + } + + var fields = partialFields + + if f.opt.AuthOwnerOnly { + fields += ",owners" + } + + fields = fmt.Sprintf("files(%s),nextPageToken", fields) + +OUTER: + for { + var files *drive.FileList + err = f.pacer.Call(func() (bool, error) { + files, err = list.Fields(googleapi.Field(fields)).Do() + return shouldRetry(err) + }) + if err != nil { + return false, errors.Wrap(err, "couldn't list directory") + } + for _, item := range files.Files { + // Convert / to ／ for listing purposes + item.Name = strings.Replace(item.Name, "/", "／", -1) + // Check the case of items is correct since + // the `=` operator is case insensitive. + + if title != "" && title != item.Name { + if stem == "" || stem != item.Name { + continue + } + _, exportName, _, _ := f.findExportFormat(item) + if exportName == "" || exportName != title { + continue + } + } + if fn(item) { + found = true + break OUTER + } + } + if files.NextPageToken == "" { + break + } + list.PageToken(files.NextPageToken) + } + return +} + +// Returns true of x is a power of 2 or zero +func isPowerOfTwo(x int64) bool { + switch { + case x == 0: + return true + case x < 0: + return false + default: + return (x & (x - 1)) == 0 + } +} + +// parseExtensions parses drive export extensions from a string +func (f *Fs) parseExtensions(extensions string) error { + for _, extension := range strings.Split(extensions, ",") { + extension = strings.ToLower(strings.TrimSpace(extension)) + if _, found := extensionToMimeType[extension]; !found { + return errors.Errorf("couldn't find mime type for extension %q", extension) + } + found := false + for _, existingExtension := range f.extensions { + if extension == existingExtension { + found = true + break + } + } + if !found { + f.extensions = append(f.extensions, extension) + } + } + return nil +} + +// Figure out if the user wants to use a team drive +func configTeamDrive(opt *Options, m configmap.Mapper, name string) error { + if opt.TeamDriveID == "" { + fmt.Printf("Configure this as a team drive?\n") + } else { + fmt.Printf("Change current team drive ID %q?\n", opt.TeamDriveID) + } + if !config.ConfirmWithDefault(false) { + return nil + } + client, err := createOAuthClient(opt, name, m) + if err != nil { + return errors.Wrap(err, "config team drive failed to create oauth client") + } + svc, err := drive.New(client) + if err != nil { + return errors.Wrap(err, "config team drive failed to make drive client") + } + fmt.Printf("Fetching team drive list...\n") + var driveIDs, driveNames []string + listTeamDrives := svc.Teamdrives.List().PageSize(100) + for { + var teamDrives *drive.TeamDriveList + err = newPacer().Call(func() (bool, error) { + teamDrives, err = listTeamDrives.Do() + return shouldRetry(err) + }) + if err != nil { + return errors.Wrap(err, "list team drives failed") + } + for _, drive := range teamDrives.TeamDrives { + driveIDs = append(driveIDs, drive.Id) + driveNames = append(driveNames, drive.Name) + } + if teamDrives.NextPageToken == "" { + break + } + listTeamDrives.PageToken(teamDrives.NextPageToken) + } + var driveID string + if len(driveIDs) == 0 { + fmt.Printf("No team drives found in your account") + } else { + driveID = config.Choose("Enter a Team Drive ID", driveIDs, driveNames, true) + } + m.Set("team_drive", driveID) + opt.TeamDriveID = driveID + return nil +} + +// newPacer makes a pacer configured for drive +func newPacer() *pacer.Pacer { + return pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer) +} + +func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) { + conf, err := google.JWTConfigFromJSON(credentialsData, driveConfig.Scopes...) + if err != nil { + return nil, errors.Wrap(err, "error processing credentials") + } + if opt.Impersonate != "" { + conf.Subject = opt.Impersonate + } + ctxWithSpecialClient := oauthutil.Context(fshttp.NewClient(fs.Config)) + return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil +} + +func createOAuthClient(opt *Options, name string, m configmap.Mapper) (*http.Client, error) { + var oAuthClient *http.Client + var err error + + // try loading service account credentials from env variable, then from a file + if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" { + loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile)) + if err != nil { + return nil, errors.Wrap(err, "error opening service account credentials file") + } + opt.ServiceAccountCredentials = string(loadedCreds) + } + if opt.ServiceAccountCredentials != "" { + oAuthClient, err = getServiceAccountClient(opt, []byte(opt.ServiceAccountCredentials)) + if err != nil { + return nil, errors.Wrap(err, "failed to create oauth client from service account") + } + } else { + oAuthClient, _, err = oauthutil.NewClient(name, m, driveConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to create oauth client") + } + } + + return oAuthClient, nil +} + +// NewFs contstructs an Fs from the path, container:path +func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + if !isPowerOfTwo(int64(opt.ChunkSize)) { + return nil, errors.Errorf("drive: chunk size %v isn't a power of two", opt.ChunkSize) + } + if opt.ChunkSize < 256*1024 { + return nil, errors.Errorf("drive: chunk size can't be less than 256k - was %v", opt.ChunkSize) + } + + oAuthClient, err := createOAuthClient(opt, name, m) + if err != nil { + return nil, errors.Wrap(err, "drive: failed when making oauth client") + } + + root, err := parseDrivePath(path) + if err != nil { + return nil, err + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + pacer: newPacer(), + } + f.isTeamDrive = opt.TeamDriveID != "" + f.features = (&fs.Features{ + DuplicateFiles: true, + ReadMimeType: true, + WriteMimeType: true, + CanHaveEmptyDirectories: true, + }).Fill(f) + + // Create a new authorized Drive client. + f.client = oAuthClient + f.svc, err = drive.New(f.client) + if err != nil { + return nil, errors.Wrap(err, "couldn't create Drive client") + } + + // set root folder for a team drive or query the user root folder + if f.isTeamDrive { + f.rootFolderID = f.opt.TeamDriveID + } else { + f.rootFolderID = "root" + } + + // override root folder if set in the config + if opt.RootFolderID != "" { + f.rootFolderID = opt.RootFolderID + } + + f.dirCache = dircache.New(root, f.rootFolderID, f) + + // Parse extensions + err = f.parseExtensions(opt.Extensions) + if err != nil { + return nil, err + } + err = f.parseExtensions(defaultExtensions) // make sure there are some sensible ones on there + if err != nil { + return nil, err + } + + // Find the current root + err = f.dirCache.FindRoot(false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + tempF := *f + tempF.dirCache = dircache.New(newRoot, f.rootFolderID, &tempF) + tempF.root = newRoot + // Make new Fs which is the parent + err = tempF.dirCache.FindRoot(false) + if err != nil { + // No root so return old f + return f, nil + } + _, err := tempF.NewObject(remote) + if err != nil { + // unable to list folder so return old f + return f, nil + } + // XXX: update the old f here instead of returning tempF, since + // `features` were already filled with functions having *f as a receiver. + // See https://github.com/ncw/rclone/issues/2182 + f.dirCache = tempF.dirCache + f.root = tempF.root + return f, fs.ErrorIsFile + } + // fmt.Printf("Root id %s", f.dirCache.RootID()) + return f, nil +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *drive.File) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + if info != nil { + o.setMetaData(info) + } else { + err := o.readMetaData() // reads info and meta, returning an error + if err != nil { + return nil, err + } + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) { + // Find the leaf in pathID + found, err = f.list([]string{pathID}, leaf, true, false, false, func(item *drive.File) bool { + if item.Name == leaf { + pathIDOut = item.Id + return true + } + if !f.opt.SkipGdocs { + _, exportName, _, _ := f.findExportFormat(item) + if exportName == leaf { + pathIDOut = item.Id + return true + } + } + return false + }) + return pathIDOut, found, err +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) { + // fmt.Println("Making", path) + // Define the metadata for the directory we are going to create. + createInfo := &drive.File{ + Name: leaf, + Description: leaf, + MimeType: driveFolderType, + Parents: []string{pathID}, + } + var info *drive.File + err = f.pacer.Call(func() (bool, error) { + info, err = f.svc.Files.Create(createInfo).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).Do() + return shouldRetry(err) + }) + if err != nil { + return "", err + } + return info.Id, nil +} + +// isAuthOwned checks if any of the item owners is the authenticated owner +func isAuthOwned(item *drive.File) bool { + for _, owner := range item.Owners { + if owner.Me { + return true + } + } + return false +} + +// exportFormats returns the export formats from drive, fetching them +// if necessary. +// +// if the fetch fails then it will not export any drive formats +func (f *Fs) exportFormats() map[string][]string { + exportFormatsOnce.Do(func() { + var about *drive.About + var err error + err = f.pacer.Call(func() (bool, error) { + about, err = f.svc.About.Get().Fields("exportFormats").Do() + return shouldRetry(err) + }) + if err != nil { + fs.Errorf(f, "Failed to get Drive exportFormats: %v", err) + _exportFormats = map[string][]string{} + return + } + _exportFormats = about.ExportFormats + }) + return _exportFormats +} + +// findExportFormat works out the optimum extension and mime-type +// for this item. +// +// Look through the extensions and find the first format that can be +// converted. If none found then return "", "" +func (f *Fs) findExportFormat(item *drive.File) (extension, filename, mimeType string, isDocument bool) { + exportMimeTypes, isDocument := f.exportFormats()[item.MimeType] + if isDocument { + for _, extension := range f.extensions { + mimeType := extensionToMimeType[extension] + for _, emt := range exportMimeTypes { + if emt == mimeType { + return extension, fmt.Sprintf("%s.%s", item.Name, extension), mimeType, true + } + } + } + } + + // else return empty + return "", "", "", isDocument +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + err = f.dirCache.FindRoot(false) + if err != nil { + return nil, err + } + directoryID, err := f.dirCache.FindDir(dir, false) + if err != nil { + return nil, err + } + + var iErr error + _, err = f.list([]string{directoryID}, "", false, false, false, func(item *drive.File) bool { + entry, err := f.itemToDirEntry(path.Join(dir, item.Name), item) + if err != nil { + return true + } + if entry != nil { + entries = append(entries, entry) + } + return false + }) + if err != nil { + return nil, err + } + if iErr != nil { + return nil, iErr + } + return entries, nil +} + +// listRRunner will read dirIDs from the in channel, perform the file listing an call cb with each DirEntry. +// +// In each cycle, will wait up to 10ms to read up to grouping entries from the in channel. +// If an error occurs it will be send to the out channel and then return. Once the in channel is closed, +// nil is send to the out channel and the function returns. +func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan string, out chan<- error, cb func(fs.DirEntry) error, grouping int) { + var dirs []string + + for dir := range in { + dirs = append(dirs[:0], dir) + wait := time.After(10 * time.Millisecond) + waitloop: + for i := 1; i < grouping; i++ { + select { + case d, ok := <-in: + if !ok { + break waitloop + } + dirs = append(dirs, d) + case <-wait: + break waitloop + } + } + var iErr error + _, err := f.list(dirs, "", false, false, false, func(item *drive.File) bool { + parentPath := "" + if len(item.Parents) > 0 { + p, ok := f.dirCache.GetInv(item.Parents[0]) + if ok { + parentPath = p + } + } + remote := path.Join(parentPath, item.Name) + entry, err := f.itemToDirEntry(remote, item) + if err != nil { + iErr = err + return true + } + + err = cb(entry) + if err != nil { + iErr = err + return true + } + return false + }) + for range dirs { + wg.Done() + } + + if iErr != nil { + out <- iErr + return + } + + if err != nil { + out <- err + return + } + } + out <- nil +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + const ( + grouping = 50 + inputBuffer = 1000 + ) + + err = f.dirCache.FindRoot(false) + if err != nil { + return err + } + directoryID, err := f.dirCache.FindDir(dir, false) + if err != nil { + return err + } + + mu := sync.Mutex{} // protects in and overflow + wg := sync.WaitGroup{} + in := make(chan string, inputBuffer) + out := make(chan error, fs.Config.Checkers) + list := walk.NewListRHelper(callback) + overfflow := []string{} + + cb := func(entry fs.DirEntry) error { + mu.Lock() + defer mu.Unlock() + if d, isDir := entry.(*fs.Dir); isDir && in != nil { + select { + case in <- d.ID(): + wg.Add(1) + default: + overfflow = append(overfflow, d.ID()) + } + } + return list.Add(entry) + } + + wg.Add(1) + in <- directoryID + + for i := 0; i < fs.Config.Checkers; i++ { + go f.listRRunner(&wg, in, out, cb, grouping) + } + go func() { + // wait until the all directories are processed + wg.Wait() + // if the input channel overflowed add the collected entries to the channel now + for len(overfflow) > 0 { + mu.Lock() + l := len(overfflow) + // only fill half of the channel to prevent entries beeing put into overfflow again + if l > inputBuffer/2 { + l = inputBuffer / 2 + } + wg.Add(l) + for _, d := range overfflow[:l] { + in <- d + } + overfflow = overfflow[l:] + mu.Unlock() + + // wait again for the completion of all directories + wg.Wait() + } + mu.Lock() + if in != nil { + // notify all workers to exit + close(in) + in = nil + } + mu.Unlock() + }() + // wait until the all workers to finish + for i := 0; i < fs.Config.Checkers; i++ { + e := <-out + mu.Lock() + // if one worker returns an error early, close the input so all other workers exit + if e != nil && in != nil { + err = e + close(in) + in = nil + } + mu.Unlock() + } + + close(out) + if err != nil { + return err + } + + return list.Flush() +} + +func (f *Fs) itemToDirEntry(remote string, item *drive.File) (fs.DirEntry, error) { + switch { + case item.MimeType == driveFolderType: + // cache the directory ID for later lookups + f.dirCache.Put(remote, item.Id) + when, _ := time.Parse(timeFormatIn, item.ModifiedTime) + d := fs.NewDir(remote, when).SetID(item.Id) + return d, nil + case f.opt.AuthOwnerOnly && !isAuthOwned(item): + // ignore object + case item.Md5Checksum != "" || item.Size > 0: + // If item has MD5 sum or a length it is a file stored on drive + o, err := f.newObjectWithInfo(remote, item) + if err != nil { + return nil, err + } + return o, nil + case f.opt.SkipGdocs: + fs.Debugf(remote, "Skipping google document type %q", item.MimeType) + default: + // If item MimeType is in the ExportFormats then it is a google doc + extension, _, exportMimeType, isDocument := f.findExportFormat(item) + if !isDocument { + fs.Debugf(remote, "Ignoring unknown document type %q", item.MimeType) + break + } + if extension == "" { + fs.Debugf(remote, "No export formats found for %q", item.MimeType) + break + } + o, err := f.newObjectWithInfo(remote+"."+extension, item) + if err != nil { + return nil, err + } + obj := o.(*Object) + obj.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, item.Id, url.QueryEscape(exportMimeType)) + if f.opt.AlternateExport { + switch item.MimeType { + case "application/vnd.google-apps.drawing": + obj.url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", item.Id, extension) + case "application/vnd.google-apps.document": + obj.url = fmt.Sprintf("https://docs.google.com/document/d/%s/export?format=%s", item.Id, extension) + case "application/vnd.google-apps.spreadsheet": + obj.url = fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=%s", item.Id, extension) + case "application/vnd.google-apps.presentation": + obj.url = fmt.Sprintf("https://docs.google.com/presentation/d/%s/export/%s", item.Id, extension) + } + } + obj.isDocument = true + obj.mimeType = exportMimeType + obj.bytes = -1 + return o, nil + } + return nil, nil +} + +// Creates a drive.File info from the parameters passed in and a half +// finished Object which must have setMetaData called on it +// +// Used to create new objects +func (f *Fs) createFileInfo(remote string, modTime time.Time, size int64) (*Object, *drive.File, error) { + // Temporary Object under construction + o := &Object{ + fs: f, + remote: remote, + bytes: size, + } + + leaf, directoryID, err := f.dirCache.FindRootAndPath(remote, true) + if err != nil { + return nil, nil, err + } + + // Define the metadata for the file we are going to create. + createInfo := &drive.File{ + Name: leaf, + Description: leaf, + Parents: []string{directoryID}, + MimeType: fs.MimeTypeFromName(remote), + ModifiedTime: modTime.Format(timeFormatOut), + } + return o, createInfo, nil +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil) + switch err { + case nil: + return exisitingObj, exisitingObj.Update(in, src, options...) + case fs.ErrorObjectNotFound: + // Not found so create it + return f.PutUnchecked(in, src, options...) + default: + return nil, err + } +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// PutUnchecked uploads the object +// +// This will create a duplicate if we upload a new file without +// checking to see if there is one already - use Put() for that. +func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime() + + o, createInfo, err := f.createFileInfo(remote, modTime, size) + if err != nil { + return nil, err + } + + var info *drive.File + if size == 0 || size < int64(f.opt.UploadCutoff) { + // Make the API request to upload metadata and file data. + // Don't retry, return a retry error instead + err = f.pacer.CallNoRetry(func() (bool, error) { + info, err = f.svc.Files.Create(createInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).KeepRevisionForever(f.opt.KeepRevisionForever).Do() + return shouldRetry(err) + }) + if err != nil { + return o, err + } + } else { + // Upload the file in chunks + info, err = f.Upload(in, size, createInfo.MimeType, "", createInfo, remote) + if err != nil { + return o, err + } + } + o.setMetaData(info) + return o, nil +} + +// MergeDirs merges the contents of all the directories passed +// in into the first one and rmdirs the other directories. +func (f *Fs) MergeDirs(dirs []fs.Directory) error { + if len(dirs) < 2 { + return nil + } + dstDir := dirs[0] + for _, srcDir := range dirs[1:] { + // list the the objects + infos := []*drive.File{} + _, err := f.list([]string{srcDir.ID()}, "", false, false, true, func(info *drive.File) bool { + infos = append(infos, info) + return false + }) + if err != nil { + return errors.Wrapf(err, "MergeDirs list failed on %v", srcDir) + } + // move them into place + for _, info := range infos { + fs.Infof(srcDir, "merging %q", info.Name) + // Move the file into the destination + err = f.pacer.Call(func() (bool, error) { + _, err = f.svc.Files.Update(info.Id, nil).RemoveParents(srcDir.ID()).AddParents(dstDir.ID()).Fields("").SupportsTeamDrives(f.isTeamDrive).Do() + return shouldRetry(err) + }) + if err != nil { + return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.Name, srcDir) + } + } + // rmdir (into trash) the now empty source directory + fs.Infof(srcDir, "removing empty directory") + err = f.rmdir(srcDir.ID(), true) + if err != nil { + return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir) + } + } + return nil +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + err := f.dirCache.FindRoot(true) + if err != nil { + return err + } + if dir != "" { + _, err = f.dirCache.FindDir(dir, true) + } + return err +} + +// Rmdir deletes a directory unconditionally by ID +func (f *Fs) rmdir(directoryID string, useTrash bool) error { + return f.pacer.Call(func() (bool, error) { + var err error + if useTrash { + info := drive.File{ + Trashed: true, + } + _, err = f.svc.Files.Update(directoryID, &info).Fields("").SupportsTeamDrives(f.isTeamDrive).Do() + } else { + err = f.svc.Files.Delete(directoryID).Fields("").SupportsTeamDrives(f.isTeamDrive).Do() + } + return shouldRetry(err) + }) +} + +// Rmdir deletes a directory +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + root := path.Join(f.root, dir) + dc := f.dirCache + directoryID, err := dc.FindDir(dir, false) + if err != nil { + return err + } + var trashedFiles = false + found, err := f.list([]string{directoryID}, "", false, false, true, func(item *drive.File) bool { + if !item.Trashed { + fs.Debugf(dir, "Rmdir: contains file: %q", item.Name) + return true + } + fs.Debugf(dir, "Rmdir: contains trashed file: %q", item.Name) + trashedFiles = true + return false + }) + if err != nil { + return err + } + if found { + return errors.Errorf("directory not empty") + } + if root != "" { + // trash the directory if it had trashed files + // in or the user wants to trash, otherwise + // delete it. + err = f.rmdir(directoryID, trashedFiles || f.opt.UseTrash) + if err != nil { + return err + } + } + f.dirCache.FlushDir(dir) + if err != nil { + return err + } + return nil +} + +// Precision of the object storage system +func (f *Fs) Precision() time.Duration { + return time.Millisecond +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + if srcObj.isDocument { + return nil, errors.New("can't copy a Google document") + } + + o, createInfo, err := f.createFileInfo(remote, srcObj.ModTime(), srcObj.bytes) + if err != nil { + return nil, err + } + + var info *drive.File + err = o.fs.pacer.Call(func() (bool, error) { + info, err = o.fs.svc.Files.Copy(srcObj.id, createInfo).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).KeepRevisionForever(f.opt.KeepRevisionForever).Do() + return shouldRetry(err) + }) + if err != nil { + return nil, err + } + + o.setMetaData(info) + return o, nil +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + if f.root == "" { + return errors.New("can't purge root directory") + } + err := f.dirCache.FindRoot(false) + if err != nil { + return err + } + err = f.pacer.Call(func() (bool, error) { + if f.opt.UseTrash { + info := drive.File{ + Trashed: true, + } + _, err = f.svc.Files.Update(f.dirCache.RootID(), &info).Fields("").SupportsTeamDrives(f.isTeamDrive).Do() + } else { + err = f.svc.Files.Delete(f.dirCache.RootID()).Fields("").SupportsTeamDrives(f.isTeamDrive).Do() + } + return shouldRetry(err) + }) + f.dirCache.ResetRoot() + if err != nil { + return err + } + return nil +} + +// CleanUp empties the trash +func (f *Fs) CleanUp() error { + err := f.pacer.Call(func() (bool, error) { + err := f.svc.Files.EmptyTrash().Do() + return shouldRetry(err) + }) + + if err != nil { + return err + } + return nil +} + +// About gets quota information +func (f *Fs) About() (*fs.Usage, error) { + if f.isTeamDrive { + // Teamdrives don't appear to have a usage API so just return empty + return &fs.Usage{}, nil + } + var about *drive.About + var err error + err = f.pacer.Call(func() (bool, error) { + about, err = f.svc.About.Get().Fields("storageQuota").Do() + return shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to get Drive storageQuota") + } + q := about.StorageQuota + usage := &fs.Usage{ + Used: fs.NewUsageValue(q.UsageInDrive), // bytes in use + Trashed: fs.NewUsageValue(q.UsageInDriveTrash), // bytes in trash + Other: fs.NewUsageValue(q.Usage - q.UsageInDrive), // other usage eg gmail in drive + } + if q.Limit > 0 { + usage.Total = fs.NewUsageValue(q.Limit) // quota of bytes that can be used + usage.Free = fs.NewUsageValue(q.Limit - q.Usage) // bytes which can be uploaded before reaching the quota + } + return usage, nil +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + if srcObj.isDocument { + return nil, errors.New("can't move a Google document") + } + _, srcParentID, err := srcObj.fs.dirCache.FindPath(src.Remote(), false) + if err != nil { + return nil, err + } + + // Temporary Object under construction + dstObj, dstInfo, err := f.createFileInfo(remote, srcObj.ModTime(), srcObj.bytes) + if err != nil { + return nil, err + } + dstParents := strings.Join(dstInfo.Parents, ",") + dstInfo.Parents = nil + + // Do the move + var info *drive.File + err = f.pacer.Call(func() (bool, error) { + info, err = f.svc.Files.Update(srcObj.id, dstInfo).RemoveParents(srcParentID).AddParents(dstParents).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).Do() + return shouldRetry(err) + }) + if err != nil { + return nil, err + } + + dstObj.setMetaData(info) + return dstObj, nil +} + +// PublicLink adds a "readable by anyone with link" permission on the given file or folder. +func (f *Fs) PublicLink(remote string) (link string, err error) { + id, err := f.dirCache.FindDir(remote, false) + if err == nil { + fs.Debugf(f, "attempting to share directory '%s'", remote) + } else { + fs.Debugf(f, "attempting to share single file '%s'", remote) + o := &Object{ + fs: f, + remote: remote, + } + if err = o.readMetaData(); err != nil { + return + } + id = o.id + } + + permission := &drive.Permission{ + AllowFileDiscovery: false, + Role: "reader", + Type: "anyone", + } + + err = f.pacer.Call(func() (bool, error) { + // TODO: On TeamDrives this might fail if lacking permissions to change ACLs. + // Need to either check `canShare` attribute on the object or see if a sufficient permission is already present. + _, err = f.svc.Permissions.Create(id, permission).Fields(googleapi.Field("id")).SupportsTeamDrives(f.isTeamDrive).Do() + return shouldRetry(err) + }) + if err != nil { + return "", err + } + return fmt.Sprintf("https://drive.google.com/open?id=%s", id), nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := path.Join(srcFs.root, srcRemote) + dstPath := path.Join(f.root, dstRemote) + + // Refuse to move to or from the root + if srcPath == "" || dstPath == "" { + fs.Debugf(src, "DirMove error: Can't move root") + return errors.New("can't move root directory") + } + + // find the root src directory + err := srcFs.dirCache.FindRoot(false) + if err != nil { + return err + } + + // find the root dst directory + if dstRemote != "" { + err = f.dirCache.FindRoot(true) + if err != nil { + return err + } + } else { + if f.dirCache.FoundRoot() { + return fs.ErrorDirExists + } + } + + // Find ID of dst parent, creating subdirs if necessary + var leaf, dstDirectoryID string + findPath := dstRemote + if dstRemote == "" { + findPath = f.root + } + leaf, dstDirectoryID, err = f.dirCache.FindPath(findPath, true) + if err != nil { + return err + } + + // Check destination does not exist + if dstRemote != "" { + _, err = f.dirCache.FindDir(dstRemote, false) + if err == fs.ErrorDirNotFound { + // OK + } else if err != nil { + return err + } else { + return fs.ErrorDirExists + } + } + + // Find ID of src parent + var srcDirectoryID string + if srcRemote == "" { + srcDirectoryID, err = srcFs.dirCache.RootParentID() + } else { + _, srcDirectoryID, err = srcFs.dirCache.FindPath(srcRemote, false) + } + if err != nil { + return err + } + + // Find ID of src + srcID, err := srcFs.dirCache.FindDir(srcRemote, false) + if err != nil { + return err + } + + // Do the move + patch := drive.File{ + Name: leaf, + } + err = f.pacer.Call(func() (bool, error) { + _, err = f.svc.Files.Update(srcID, &patch).RemoveParents(srcDirectoryID).AddParents(dstDirectoryID).Fields("").SupportsTeamDrives(f.isTeamDrive).Do() + return shouldRetry(err) + }) + if err != nil { + return err + } + srcFs.dirCache.FlushDir(srcRemote) + return nil +} + +// ChangeNotify calls the passed function with a path that has had changes. +// If the implementation uses polling, it should adhere to the given interval. +// +// Automatically restarts itself in case of unexpected behaviour of the remote. +// +// Close the returned channel to stop being notified. +func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool { + quit := make(chan bool) + go func() { + select { + case <-quit: + return + default: + for { + f.changeNotifyRunner(notifyFunc, pollInterval) + fs.Debugf(f, "Notify listener service ran into issues, restarting shortly.") + time.Sleep(pollInterval) + } + } + }() + return quit +} + +func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) { + var err error + var startPageToken *drive.StartPageToken + err = f.pacer.Call(func() (bool, error) { + startPageToken, err = f.svc.Changes.GetStartPageToken().SupportsTeamDrives(f.isTeamDrive).Do() + return shouldRetry(err) + }) + if err != nil { + fs.Debugf(f, "Failed to get StartPageToken: %v", err) + return + } + pageToken := startPageToken.StartPageToken + + for { + fs.Debugf(f, "Checking for changes on remote") + var changeList *drive.ChangeList + + err = f.pacer.Call(func() (bool, error) { + changesCall := f.svc.Changes.List(pageToken).Fields("nextPageToken,newStartPageToken,changes(fileId,file(name,parents,mimeType))") + if f.opt.ListChunk > 0 { + changesCall.PageSize(f.opt.ListChunk) + } + if f.isTeamDrive { + changesCall.TeamDriveId(f.opt.TeamDriveID) + changesCall.SupportsTeamDrives(true) + changesCall.IncludeTeamDriveItems(true) + } + changeList, err = changesCall.Do() + return shouldRetry(err) + }) + if err != nil { + fs.Debugf(f, "Failed to get Changes: %v", err) + return + } + + type entryType struct { + path string + entryType fs.EntryType + } + var pathsToClear []entryType + for _, change := range changeList.Changes { + if path, ok := f.dirCache.GetInv(change.FileId); ok { + if change.File != nil && change.File.MimeType != driveFolderType { + pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryObject}) + } else { + pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryDirectory}) + } + continue + } + + if change.File != nil { + changeType := fs.EntryDirectory + if change.File.MimeType != driveFolderType { + changeType = fs.EntryObject + } + + // translate the parent dir of this object + if len(change.File.Parents) > 0 { + if path, ok := f.dirCache.GetInv(change.File.Parents[0]); ok { + // and append the drive file name to compute the full file name + if len(path) > 0 { + path = path + "/" + change.File.Name + } else { + path = change.File.Name + } + // this will now clear the actual file too + pathsToClear = append(pathsToClear, entryType{path: path, entryType: changeType}) + } + } else { // a true root object that is changed + pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: changeType}) + } + } + } + + visitedPaths := make(map[string]bool) + for _, entry := range pathsToClear { + if _, ok := visitedPaths[entry.path]; ok { + continue + } + visitedPaths[entry.path] = true + notifyFunc(entry.path, entry.entryType) + } + + if changeList.NewStartPageToken != "" { + pageToken = changeList.NewStartPageToken + fs.Debugf(f, "All changes were processed. Waiting for more.") + time.Sleep(pollInterval) + } else if changeList.NextPageToken != "" { + pageToken = changeList.NextPageToken + fs.Debugf(f, "There are more changes pending, checking now.") + } else { + fs.Debugf(f, "Did not get any page token, something went wrong! %+v", changeList) + return + } + } +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *Fs) DirCacheFlush() { + f.dirCache.ResetRoot() +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the Md5sum of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.MD5 { + return "", hash.ErrUnsupported + } + return o.md5sum, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.bytes +} + +// setMetaData sets the fs data from a drive.File +func (o *Object) setMetaData(info *drive.File) { + o.id = info.Id + o.url = fmt.Sprintf("%sfiles/%s?alt=media", o.fs.svc.BasePath, info.Id) + o.md5sum = strings.ToLower(info.Md5Checksum) + o.bytes = info.Size + if o.fs.opt.UseCreatedDate { + o.modifiedDate = info.CreatedTime + } else { + o.modifiedDate = info.ModifiedTime + } + o.mimeType = info.MimeType +} + +// setGdocsMetaData only sets the gdocs related fields +func (o *Object) setGdocsMetaData(info *drive.File, extension, exportMimeType string) { + o.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", o.fs.svc.BasePath, info.Id, url.QueryEscape(exportMimeType)) + if o.fs.opt.AlternateExport { + switch info.MimeType { + case "application/vnd.google-apps.drawing": + o.url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", info.Id, extension) + case "application/vnd.google-apps.document": + o.url = fmt.Sprintf("https://docs.google.com/document/d/%s/export?format=%s", info.Id, extension) + case "application/vnd.google-apps.spreadsheet": + o.url = fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=%s", info.Id, extension) + case "application/vnd.google-apps.presentation": + o.url = fmt.Sprintf("https://docs.google.com/presentation/d/%s/export/%s", info.Id, extension) + } + } + o.isDocument = true + o.mimeType = exportMimeType + o.bytes = -1 +} + +// readMetaData gets the info if it hasn't already been fetched +func (o *Object) readMetaData() (err error) { + if o.id != "" { + return nil + } + + leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(o.remote, false) + if err != nil { + if err == fs.ErrorDirNotFound { + return fs.ErrorObjectNotFound + } + return err + } + + found, err := o.fs.list([]string{directoryID}, leaf, false, true, false, func(item *drive.File) bool { + if item.Name == leaf { + o.setMetaData(item) + return true + } + if !o.fs.opt.SkipGdocs { + extension, exportName, exportMimeType, _ := o.fs.findExportFormat(item) + if exportName == leaf { + o.setMetaData(item) + o.setGdocsMetaData(item, extension, exportMimeType) + return true + } + } + return false + }) + if err != nil { + return err + } + if !found { + return fs.ErrorObjectNotFound + } + return nil +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Debugf(o, "Failed to read metadata: %v", err) + return time.Now() + } + modTime, err := time.Parse(timeFormatIn, o.modifiedDate) + if err != nil { + fs.Debugf(o, "Failed to read mtime from object: %v", err) + return time.Now() + } + return modTime +} + +// SetModTime sets the modification time of the drive fs object +func (o *Object) SetModTime(modTime time.Time) error { + err := o.readMetaData() + if err != nil { + return err + } + // New metadata + updateInfo := &drive.File{ + ModifiedTime: modTime.Format(timeFormatOut), + } + // Set modified date + var info *drive.File + err = o.fs.pacer.Call(func() (bool, error) { + info, err = o.fs.svc.Files.Update(o.id, updateInfo).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(o.fs.isTeamDrive).Do() + return shouldRetry(err) + }) + if err != nil { + return err + } + // Update info from read data + o.setMetaData(info) + return nil +} + +// Storable returns a boolean as to whether this object is storable +func (o *Object) Storable() bool { + return true +} + +// httpResponse gets an http.Response object for the object o.url +// using the method passed in +func (o *Object) httpResponse(method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) { + if o.url == "" { + return nil, nil, errors.New("forbidden to download - check sharing permission") + } + if o.isDocument { + for _, o := range options { + // https://developers.google.com/drive/v3/web/manage-downloads#partial_download + if _, ok := o.(*fs.RangeOption); ok { + return nil, nil, errors.New("partial downloads are not supported while exporting Google Documents") + } + } + } + req, err = http.NewRequest(method, o.url, nil) + if err != nil { + return req, nil, err + } + fs.OpenOptionAddHTTPHeaders(req.Header, options) + err = o.fs.pacer.Call(func() (bool, error) { + res, err = o.fs.client.Do(req) + if err == nil { + err = googleapi.CheckResponse(res) + if err != nil { + _ = res.Body.Close() // ignore error + } + } + return shouldRetry(err) + }) + if err != nil { + return req, nil, err + } + return req, res, nil +} + +// openFile represents an Object open for reading +type openFile struct { + o *Object // Object we are reading for + in io.ReadCloser // reading from here + bytes int64 // number of bytes read on this connection + eof bool // whether we have read end of file + errored bool // whether we have encountered an error during reading +} + +// Read bytes from the object - see io.Reader +func (file *openFile) Read(p []byte) (n int, err error) { + n, err = file.in.Read(p) + file.bytes += int64(n) + if err != nil && err != io.EOF { + file.errored = true + } + if err == io.EOF { + file.eof = true + } + return +} + +// Close the object and update bytes read +func (file *openFile) Close() (err error) { + // If end of file, update bytes read + if file.eof && !file.errored { + fs.Debugf(file.o, "Updating size of doc after download to %v", file.bytes) + file.o.bytes = file.bytes + } + return file.in.Close() +} + +// Check it satisfies the interfaces +var _ io.ReadCloser = &openFile{} + +// Checks to see if err is a googleapi.Error with of type what +func isGoogleError(err error, what string) bool { + if gerr, ok := err.(*googleapi.Error); ok { + for _, error := range gerr.Errors { + if error.Reason == what { + return true + } + } + } + return false +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + _, res, err := o.httpResponse("GET", options) + if err != nil { + if isGoogleError(err, "cannotDownloadAbusiveFile") { + if o.fs.opt.AcknowledgeAbuse { + // Retry acknowledging abuse + if strings.ContainsRune(o.url, '?') { + o.url += "&" + } else { + o.url += "?" + } + o.url += "acknowledgeAbuse=true" + _, res, err = o.httpResponse("GET", options) + } else { + err = errors.Wrap(err, "Use the --drive-acknowledge-abuse flag to download this file") + } + } + if err != nil { + return nil, errors.Wrap(err, "open file failed") + } + } + // If it is a document, update the size with what we are + // reading as it can change from the HEAD in the listing to + // this GET. This stops rclone marking the transfer as + // corrupted. + if o.isDocument { + return &openFile{o: o, in: res.Body}, nil + } + return res.Body, nil +} + +// Update the already existing object +// +// Copy the reader into the object updating modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + size := src.Size() + modTime := src.ModTime() + if o.isDocument { + return errors.New("can't update a google document") + } + updateInfo := &drive.File{ + MimeType: fs.MimeType(src), + ModifiedTime: modTime.Format(timeFormatOut), + } + + // Make the API request to upload metadata and file data. + var err error + var info *drive.File + if size == 0 || size < int64(o.fs.opt.UploadCutoff) { + // Don't retry, return a retry error instead + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + info, err = o.fs.svc.Files.Update(o.id, updateInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(o.fs.isTeamDrive).KeepRevisionForever(o.fs.opt.KeepRevisionForever).Do() + return shouldRetry(err) + }) + if err != nil { + return err + } + } else { + // Upload the file in chunks + info, err = o.fs.Upload(in, size, updateInfo.MimeType, o.id, updateInfo, o.remote) + if err != nil { + return err + } + } + o.setMetaData(info) + return nil +} + +// Remove an object +func (o *Object) Remove() error { + if o.isDocument { + return errors.New("can't delete a google document") + } + var err error + err = o.fs.pacer.Call(func() (bool, error) { + if o.fs.opt.UseTrash { + info := drive.File{ + Trashed: true, + } + _, err = o.fs.svc.Files.Update(o.id, &info).Fields("").SupportsTeamDrives(o.fs.isTeamDrive).Do() + } else { + err = o.fs.svc.Files.Delete(o.id).Fields("").SupportsTeamDrives(o.fs.isTeamDrive).Do() + } + return shouldRetry(err) + }) + return err +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + err := o.readMetaData() + if err != nil { + fs.Debugf(o, "Failed to read metadata: %v", err) + return "" + } + return o.mimeType +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.id +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.CleanUpper = (*Fs)(nil) + _ fs.PutStreamer = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.ChangeNotifier = (*Fs)(nil) + _ fs.PutUncheckeder = (*Fs)(nil) + _ fs.PublicLinker = (*Fs)(nil) + _ fs.ListRer = (*Fs)(nil) + _ fs.MergeDirser = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = (*Object)(nil) + _ fs.IDer = (*Object)(nil) +) diff --git a/.rclone_repo/backend/drive/drive_internal_test.go b/.rclone_repo/backend/drive/drive_internal_test.go new file mode 100755 index 0000000..5040c9f --- /dev/null +++ b/.rclone_repo/backend/drive/drive_internal_test.go @@ -0,0 +1,119 @@ +package drive + +import ( + "encoding/json" + "testing" + + "google.golang.org/api/drive/v3" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +const exampleExportFormats = `{ + "application/vnd.google-apps.document": [ + "application/rtf", + "application/vnd.oasis.opendocument.text", + "text/html", + "application/pdf", + "application/epub+zip", + "application/zip", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "text/plain" + ], + "application/vnd.google-apps.spreadsheet": [ + "application/x-vnd.oasis.opendocument.spreadsheet", + "text/tab-separated-values", + "application/pdf", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "text/csv", + "application/zip", + "application/vnd.oasis.opendocument.spreadsheet" + ], + "application/vnd.google-apps.jam": [ + "application/pdf" + ], + "application/vnd.google-apps.script": [ + "application/vnd.google-apps.script+json" + ], + "application/vnd.google-apps.presentation": [ + "application/vnd.oasis.opendocument.presentation", + "application/pdf", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "text/plain" + ], + "application/vnd.google-apps.form": [ + "application/zip" + ], + "application/vnd.google-apps.drawing": [ + "image/svg+xml", + "image/png", + "application/pdf", + "image/jpeg" + ] +}` + +// Load the example export formats into exportFormats for testing +func TestInternalLoadExampleExportFormats(t *testing.T) { + exportFormatsOnce.Do(func() {}) + assert.NoError(t, json.Unmarshal([]byte(exampleExportFormats), &_exportFormats)) +} + +func TestInternalParseExtensions(t *testing.T) { + for _, test := range []struct { + in string + want []string + wantErr error + }{ + {"doc", []string{"doc"}, nil}, + {" docx ,XLSX, pptx,svg", []string{"docx", "xlsx", "pptx", "svg"}, nil}, + {"docx,svg,Docx", []string{"docx", "svg"}, nil}, + {"docx,potato,docx", []string{"docx"}, errors.New(`couldn't find mime type for extension "potato"`)}, + } { + f := new(Fs) + gotErr := f.parseExtensions(test.in) + if test.wantErr == nil { + assert.NoError(t, gotErr) + } else { + assert.EqualError(t, gotErr, test.wantErr.Error()) + } + assert.Equal(t, test.want, f.extensions) + } + + // Test it is appending + f := new(Fs) + assert.Nil(t, f.parseExtensions("docx,svg")) + assert.Nil(t, f.parseExtensions("docx,svg,xlsx")) + assert.Equal(t, []string{"docx", "svg", "xlsx"}, f.extensions) + +} + +func TestInternalFindExportFormat(t *testing.T) { + item := &drive.File{ + Name: "file", + MimeType: "application/vnd.google-apps.document", + } + for _, test := range []struct { + extensions []string + wantExtension string + wantMimeType string + }{ + {[]string{}, "", ""}, + {[]string{"pdf"}, "pdf", "application/pdf"}, + {[]string{"pdf", "rtf", "xls"}, "pdf", "application/pdf"}, + {[]string{"xls", "rtf", "pdf"}, "rtf", "application/rtf"}, + {[]string{"xls", "csv", "svg"}, "", ""}, + } { + f := new(Fs) + f.extensions = test.extensions + gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(item) + assert.Equal(t, test.wantExtension, gotExtension) + if test.wantExtension != "" { + assert.Equal(t, item.Name+"."+gotExtension, gotFilename) + } else { + assert.Equal(t, "", gotFilename) + } + assert.Equal(t, test.wantMimeType, gotMimeType) + assert.Equal(t, true, gotIsDocument) + } +} diff --git a/.rclone_repo/backend/drive/drive_test.go b/.rclone_repo/backend/drive/drive_test.go new file mode 100755 index 0000000..ec3a612 --- /dev/null +++ b/.rclone_repo/backend/drive/drive_test.go @@ -0,0 +1,17 @@ +// Test Drive filesystem interface +package drive_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/drive" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestDrive:", + NilObject: (*drive.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/drive/upload.go b/.rclone_repo/backend/drive/upload.go new file mode 100755 index 0000000..350637d --- /dev/null +++ b/.rclone_repo/backend/drive/upload.go @@ -0,0 +1,249 @@ +// Upload for drive +// +// Docs +// Resumable upload: https://developers.google.com/drive/web/manage-uploads#resumable +// Best practices: https://developers.google.com/drive/web/manage-uploads#best-practices +// Files insert: https://developers.google.com/drive/v2/reference/files/insert +// Files update: https://developers.google.com/drive/v2/reference/files/update +// +// This contains code adapted from google.golang.org/api (C) the GO AUTHORS + +package drive + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strconv" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/lib/readers" + "github.com/pkg/errors" + "google.golang.org/api/drive/v3" + "google.golang.org/api/googleapi" +) + +const ( + // statusResumeIncomplete is the code returned by the Google uploader when the transfer is not yet complete. + statusResumeIncomplete = 308 +) + +// resumableUpload is used by the generated APIs to provide resumable uploads. +// It is not used by developers directly. +type resumableUpload struct { + f *Fs + remote string + // URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable". + URI string + // Media is the object being uploaded. + Media io.Reader + // MediaType defines the media type, e.g. "image/jpeg". + MediaType string + // ContentLength is the full size of the object being uploaded. + ContentLength int64 + // Return value + ret *drive.File +} + +// Upload the io.Reader in of size bytes with contentType and info +func (f *Fs) Upload(in io.Reader, size int64, contentType string, fileID string, info *drive.File, remote string) (*drive.File, error) { + params := make(url.Values) + params.Set("alt", "json") + params.Set("uploadType", "resumable") + params.Set("fields", partialFields) + if f.isTeamDrive { + params.Set("supportsTeamDrives", "true") + } + if f.opt.KeepRevisionForever { + params.Set("keepRevisionForever", "true") + } + urls := "https://www.googleapis.com/upload/drive/v3/files" + method := "POST" + if fileID != "" { + params.Set("setModifiedDate", "true") + urls += "/{fileId}" + method = "PATCH" + } + urls += "?" + params.Encode() + var res *http.Response + var err error + err = f.pacer.Call(func() (bool, error) { + var body io.Reader + body, err = googleapi.WithoutDataWrapper.JSONReader(info) + if err != nil { + return false, err + } + var req *http.Request + req, err = http.NewRequest(method, urls, body) + if err != nil { + return false, err + } + googleapi.Expand(req.URL, map[string]string{ + "fileId": fileID, + }) + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + req.Header.Set("X-Upload-Content-Type", contentType) + req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size)) + res, err = f.client.Do(req) + if err == nil { + defer googleapi.CloseBody(res) + err = googleapi.CheckResponse(res) + } + return shouldRetry(err) + }) + if err != nil { + return nil, err + } + loc := res.Header.Get("Location") + rx := &resumableUpload{ + f: f, + remote: remote, + URI: loc, + Media: in, + MediaType: contentType, + ContentLength: size, + } + return rx.Upload() +} + +// Make an http.Request for the range passed in +func (rx *resumableUpload) makeRequest(start int64, body io.ReadSeeker, reqSize int64) *http.Request { + req, _ := http.NewRequest("POST", rx.URI, body) + req.ContentLength = reqSize + if reqSize != 0 { + req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength)) + } else { + req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", rx.ContentLength)) + } + req.Header.Set("Content-Type", rx.MediaType) + return req +} + +// rangeRE matches the transfer status response from the server. $1 is +// the last byte index uploaded. +var rangeRE = regexp.MustCompile(`^0\-(\d+)$`) + +// Query drive for the amount transferred so far +// +// If error is nil, then start should be valid +func (rx *resumableUpload) transferStatus() (start int64, err error) { + req := rx.makeRequest(0, nil, 0) + res, err := rx.f.client.Do(req) + if err != nil { + return 0, err + } + defer googleapi.CloseBody(res) + if res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusOK { + return rx.ContentLength, nil + } + if res.StatusCode != statusResumeIncomplete { + err = googleapi.CheckResponse(res) + if err != nil { + return 0, err + } + return 0, errors.Errorf("unexpected http return code %v", res.StatusCode) + } + Range := res.Header.Get("Range") + if m := rangeRE.FindStringSubmatch(Range); len(m) == 2 { + start, err = strconv.ParseInt(m[1], 10, 64) + if err == nil { + return start, nil + } + } + return 0, errors.Errorf("unable to parse range %q", Range) +} + +// Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil +func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) { + _, _ = chunk.Seek(0, io.SeekStart) + req := rx.makeRequest(start, chunk, chunkSize) + res, err := rx.f.client.Do(req) + if err != nil { + return 599, err + } + defer googleapi.CloseBody(res) + if res.StatusCode == statusResumeIncomplete { + return res.StatusCode, nil + } + err = googleapi.CheckResponse(res) + if err != nil { + return res.StatusCode, err + } + + // When the entire file upload is complete, the server + // responds with an HTTP 201 Created along with any metadata + // associated with this resource. If this request had been + // updating an existing entity rather than creating a new one, + // the HTTP response code for a completed upload would have + // been 200 OK. + // + // So parse the response out of the body. We aren't expecting + // any other 2xx codes, so we parse it unconditionaly on + // StatusCode + if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil { + return 598, err + } + + return res.StatusCode, nil +} + +// Upload uploads the chunks from the input +// It retries each chunk using the pacer and --low-level-retries +func (rx *resumableUpload) Upload() (*drive.File, error) { + start := int64(0) + var StatusCode int + var err error + buf := make([]byte, int(rx.f.opt.ChunkSize)) + for start < rx.ContentLength { + reqSize := rx.ContentLength - start + if reqSize >= int64(rx.f.opt.ChunkSize) { + reqSize = int64(rx.f.opt.ChunkSize) + } + chunk := readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize) + + // Transfer the chunk + err = rx.f.pacer.Call(func() (bool, error) { + fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize) + StatusCode, err = rx.transferChunk(start, chunk, reqSize) + again, err := shouldRetry(err) + if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK { + again = false + err = nil + } + return again, err + }) + if err != nil { + return nil, err + } + + start += reqSize + } + // Resume or retry uploads that fail due to connection interruptions or + // any 5xx errors, including: + // + // 500 Internal Server Error + // 502 Bad Gateway + // 503 Service Unavailable + // 504 Gateway Timeout + // + // Use an exponential backoff strategy if any 5xx server error is + // returned when resuming or retrying upload requests. These errors can + // occur if a server is getting overloaded. Exponential backoff can help + // alleviate these kinds of problems during periods of high volume of + // requests or heavy network traffic. Other kinds of requests should not + // be handled by exponential backoff but you can still retry a number of + // them. When retrying these requests, limit the number of times you + // retry them. For example your code could limit to ten retries or less + // before reporting an error. + // + // Handle 404 Not Found errors when doing resumable uploads by starting + // the entire upload over from the beginning. + if rx.ret == nil { + return nil, fserrors.RetryErrorf("Incomplete upload - retry, last error %d", StatusCode) + } + return rx.ret, nil +} diff --git a/.rclone_repo/backend/dropbox/dbhash/dbhash.go b/.rclone_repo/backend/dropbox/dbhash/dbhash.go new file mode 100755 index 0000000..3cdc5d7 --- /dev/null +++ b/.rclone_repo/backend/dropbox/dbhash/dbhash.go @@ -0,0 +1,127 @@ +// Package dbhash implements the dropbox hash as described in +// +// https://www.dropbox.com/developers/reference/content-hash +package dbhash + +import ( + "crypto/sha256" + "hash" +) + +const ( + // BlockSize of the checksum in bytes. + BlockSize = sha256.BlockSize + // Size of the checksum in bytes. + Size = sha256.BlockSize + bytesPerBlock = 4 * 1024 * 1024 + hashReturnedError = "hash function returned error" +) + +type digest struct { + n int // bytes written into blockHash so far + blockHash hash.Hash + totalHash hash.Hash + sumCalled bool + writtenMore bool +} + +// New returns a new hash.Hash computing the Dropbox checksum. +func New() hash.Hash { + d := &digest{} + d.Reset() + return d +} + +// writeBlockHash writes the current block hash into the total hash +func (d *digest) writeBlockHash() { + blockHash := d.blockHash.Sum(nil) + _, err := d.totalHash.Write(blockHash) + if err != nil { + panic(hashReturnedError) + } + // reset counters for blockhash + d.n = 0 + d.blockHash.Reset() +} + +// Write writes len(p) bytes from p to the underlying data stream. It returns +// the number of bytes written from p (0 <= n <= len(p)) and any error +// encountered that caused the write to stop early. Write must return a non-nil +// error if it returns n < len(p). Write must not modify the slice data, even +// temporarily. +// +// Implementations must not retain p. +func (d *digest) Write(p []byte) (n int, err error) { + n = len(p) + for len(p) > 0 { + d.writtenMore = true + toWrite := bytesPerBlock - d.n + if toWrite > len(p) { + toWrite = len(p) + } + _, err = d.blockHash.Write(p[:toWrite]) + if err != nil { + panic(hashReturnedError) + } + d.n += toWrite + p = p[toWrite:] + // Accumulate the total hash + if d.n == bytesPerBlock { + d.writeBlockHash() + } + } + return n, nil +} + +// Sum appends the current hash to b and returns the resulting slice. +// It does not change the underlying hash state. +// +// TODO(ncw) Sum() can only be called once for this type of hash. +// If you call Sum(), then Write() then Sum() it will result in +// a panic. Calling Write() then Sum(), then Sum() is OK. +func (d *digest) Sum(b []byte) []byte { + if d.sumCalled && d.writtenMore { + panic("digest.Sum() called more than once") + } + d.sumCalled = true + d.writtenMore = false + if d.n != 0 { + d.writeBlockHash() + } + return d.totalHash.Sum(b) +} + +// Reset resets the Hash to its initial state. +func (d *digest) Reset() { + d.n = 0 + d.totalHash = sha256.New() + d.blockHash = sha256.New() + d.sumCalled = false + d.writtenMore = false +} + +// Size returns the number of bytes Sum will return. +func (d *digest) Size() int { + return d.totalHash.Size() +} + +// BlockSize returns the hash's underlying block size. +// The Write method must be able to accept any amount +// of data, but it may operate more efficiently if all writes +// are a multiple of the block size. +func (d *digest) BlockSize() int { + return d.totalHash.BlockSize() +} + +// Sum returns the Dropbox checksum of the data. +func Sum(data []byte) [Size]byte { + var d digest + d.Reset() + _, _ = d.Write(data) + var out [Size]byte + d.Sum(out[:0]) + return out +} + +// must implement this interface +var _ hash.Hash = (*digest)(nil) diff --git a/.rclone_repo/backend/dropbox/dbhash/dbhash_test.go b/.rclone_repo/backend/dropbox/dbhash/dbhash_test.go new file mode 100755 index 0000000..9bac957 --- /dev/null +++ b/.rclone_repo/backend/dropbox/dbhash/dbhash_test.go @@ -0,0 +1,88 @@ +package dbhash_test + +import ( + "encoding/hex" + "fmt" + "testing" + + "github.com/ncw/rclone/backend/dropbox/dbhash" + "github.com/stretchr/testify/assert" +) + +func testChunk(t *testing.T, chunk int) { + data := make([]byte, chunk) + for i := 0; i < chunk; i++ { + data[i] = 'A' + } + for _, test := range []struct { + n int + want string + }{ + {0, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + {1, "1cd6ef71e6e0ff46ad2609d403dc3fee244417089aa4461245a4e4fe23a55e42"}, + {2, "01e0655fb754d10418a73760f57515f4903b298e6d67dda6bf0987fa79c22c88"}, + {4096, "8620913d33852befe09f16fff8fd75f77a83160d29f76f07e0276e9690903035"}, + {4194303, "647c8627d70f7a7d13ce96b1e7710a771a55d41a62c3da490d92e56044d311fa"}, + {4194304, "d4d63bac5b866c71620185392a8a6218ac1092454a2d16f820363b69852befa3"}, + {4194305, "8f553da8d00d0bf509d8470e242888be33019c20c0544811f5b2b89e98360b92"}, + {8388607, "83b30cf4fb5195b04a937727ae379cf3d06673bf8f77947f6a92858536e8369c"}, + {8388608, "e08b3ba1f538804075c5f939accdeaa9efc7b5c01865c94a41e78ca6550a88e7"}, + {8388609, "02c8a4aefc2bfc9036f89a7098001865885938ca580e5c9e5db672385edd303c"}, + } { + d := dbhash.New() + var toWrite int + for toWrite = test.n; toWrite >= chunk; toWrite -= chunk { + n, err := d.Write(data) + assert.Nil(t, err) + assert.Equal(t, chunk, n) + } + n, err := d.Write(data[:toWrite]) + assert.Nil(t, err) + assert.Equal(t, toWrite, n) + got := hex.EncodeToString(d.Sum(nil)) + assert.Equal(t, test.want, got, fmt.Sprintf("when testing length %d", n)) + + } +} + +func TestHashChunk16M(t *testing.T) { testChunk(t, 16*1024*1024) } +func TestHashChunk8M(t *testing.T) { testChunk(t, 8*1024*1024) } +func TestHashChunk4M(t *testing.T) { testChunk(t, 4*1024*1024) } +func TestHashChunk2M(t *testing.T) { testChunk(t, 2*1024*1024) } +func TestHashChunk1M(t *testing.T) { testChunk(t, 1*1024*1024) } +func TestHashChunk64k(t *testing.T) { testChunk(t, 64*1024) } +func TestHashChunk32k(t *testing.T) { testChunk(t, 32*1024) } +func TestHashChunk2048(t *testing.T) { testChunk(t, 2048) } +func TestHashChunk2047(t *testing.T) { testChunk(t, 2047) } + +func TestSumCalledTwice(t *testing.T) { + d := dbhash.New() + assert.NotPanics(t, func() { d.Sum(nil) }) + d.Reset() + assert.NotPanics(t, func() { d.Sum(nil) }) + assert.NotPanics(t, func() { d.Sum(nil) }) + _, _ = d.Write([]byte{1}) + assert.Panics(t, func() { d.Sum(nil) }) +} + +func TestSize(t *testing.T) { + d := dbhash.New() + assert.Equal(t, 32, d.Size()) +} + +func TestBlockSize(t *testing.T) { + d := dbhash.New() + assert.Equal(t, 64, d.BlockSize()) +} + +func TestSum(t *testing.T) { + assert.Equal(t, + [64]byte{ + 0x1c, 0xd6, 0xef, 0x71, 0xe6, 0xe0, 0xff, 0x46, + 0xad, 0x26, 0x09, 0xd4, 0x03, 0xdc, 0x3f, 0xee, + 0x24, 0x44, 0x17, 0x08, 0x9a, 0xa4, 0x46, 0x12, + 0x45, 0xa4, 0xe4, 0xfe, 0x23, 0xa5, 0x5e, 0x42, + }, + dbhash.Sum([]byte{'A'}), + ) +} diff --git a/.rclone_repo/backend/dropbox/dropbox.go b/.rclone_repo/backend/dropbox/dropbox.go new file mode 100755 index 0000000..87686bb --- /dev/null +++ b/.rclone_repo/backend/dropbox/dropbox.go @@ -0,0 +1,1082 @@ +// Package dropbox provides an interface to Dropbox object storage +package dropbox + +// FIXME dropbox for business would be quite easy to add + +/* +The Case folding of PathDisplay problem + +From the docs: + +path_display String. The cased path to be used for display purposes +only. In rare instances the casing will not correctly match the user's +filesystem, but this behavior will match the path provided in the Core +API v1, and at least the last path component will have the correct +casing. Changes to only the casing of paths won't be returned by +list_folder/continue. This field will be null if the file or folder is +not mounted. This field is optional. + +We solve this by not implementing the ListR interface. The dropbox +remote will recurse directory by directory only using the last element +of path_display and all will be well. +*/ + +import ( + "fmt" + "io" + "log" + "path" + "regexp" + "strings" + "time" + + "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox" + "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/common" + "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files" + "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/sharing" + "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/oauthutil" + "github.com/ncw/rclone/lib/pacer" + "github.com/ncw/rclone/lib/readers" + "github.com/pkg/errors" + "golang.org/x/oauth2" +) + +// Constants +const ( + rcloneClientID = "5jcck7diasz0rqy" + rcloneEncryptedClientSecret = "fRS5vVLr2v6FbyXYnIgjwBuUAt0osq_QZTXAEcmZ7g" + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential + // Upload chunk size - setting too small makes uploads slow. + // Chunks are buffered into memory for retries. + // + // Speed vs chunk size uploading a 1 GB file on 2017-11-22 + // + // Chunk Size MB, Speed Mbyte/s, % of max + // 1 1.364 11% + // 2 2.443 19% + // 4 4.288 33% + // 8 6.79 52% + // 16 8.916 69% + // 24 10.195 79% + // 32 10.427 81% + // 40 10.96 85% + // 48 11.828 91% + // 56 11.763 91% + // 64 12.047 93% + // 96 12.302 95% + // 128 12.945 100% + // + // Choose 48MB which is 91% of Maximum speed. rclone by + // default does 4 transfers so this should use 4*48MB = 192MB + // by default. + defaultChunkSize = 48 * 1024 * 1024 + maxChunkSize = 150 * 1024 * 1024 +) + +var ( + // Description of how to auth for this app + dropboxConfig = &oauth2.Config{ + Scopes: []string{}, + // Endpoint: oauth2.Endpoint{ + // AuthURL: "https://www.dropbox.com/1/oauth2/authorize", + // TokenURL: "https://api.dropboxapi.com/1/oauth2/token", + // }, + Endpoint: dropbox.OAuthEndpoint(""), + ClientID: rcloneClientID, + ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), + RedirectURL: oauthutil.RedirectLocalhostURL, + } + // A regexp matching path names for files Dropbox ignores + // See https://www.dropbox.com/en/help/145 - Ignored files + ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`) +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "dropbox", + Description: "Dropbox", + NewFs: NewFs, + Config: func(name string, m configmap.Mapper) { + err := oauthutil.ConfigNoOffline("dropbox", name, m, dropboxConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + }, + Options: []fs.Option{{ + Name: config.ConfigClientID, + Help: "Dropbox App Client Id\nLeave blank normally.", + }, { + Name: config.ConfigClientSecret, + Help: "Dropbox App Client Secret\nLeave blank normally.", + }, { + Name: "chunk_size", + Help: fmt.Sprintf("Upload chunk size. Max %v.", fs.SizeSuffix(maxChunkSize)), + Default: fs.SizeSuffix(defaultChunkSize), + Advanced: true, + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + ChunkSize fs.SizeSuffix `config:"chunk_size"` +} + +// Fs represents a remote dropbox server +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + features *fs.Features // optional features + srv files.Client // the connection to the dropbox server + sharing sharing.Client // as above, but for generating sharing links + users users.Client // as above, but for accessing user information + slashRoot string // root with "/" prefix, lowercase + slashRootSlash string // root with "/" prefix and postfix, lowercase + pacer *pacer.Pacer // To pace the API calls + ns string // The namespace we are using or "" for none +} + +// Object describes a dropbox object +// +// Dropbox Objects always have full metadata +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + bytes int64 // size of the object + modTime time.Time // time it was last modified + hash string // content_hash of the object +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("Dropbox root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// shouldRetry returns a boolean as to whether this err deserves to be +// retried. It returns the err as a convenience +func shouldRetry(err error) (bool, error) { + if err == nil { + return false, err + } + baseErrString := errors.Cause(err).Error() + // FIXME there is probably a better way of doing this! + if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") { + return true, err + } + return fserrors.ShouldRetry(err), err +} + +// NewFs contstructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + if opt.ChunkSize > maxChunkSize { + return nil, errors.Errorf("chunk size too big, must be < %v", maxChunkSize) + } + + // Convert the old token if it exists. The old token was just + // just a string, the new one is a JSON blob + oldToken, ok := m.Get(config.ConfigToken) + oldToken = strings.TrimSpace(oldToken) + if ok && oldToken != "" && oldToken[0] != '{' { + fs.Infof(name, "Converting token to new format") + newToken := fmt.Sprintf(`{"access_token":"%s","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}`, oldToken) + err := config.SetValueAndSave(name, config.ConfigToken, newToken) + if err != nil { + return nil, errors.Wrap(err, "NewFS convert token") + } + } + + oAuthClient, _, err := oauthutil.NewClient(name, m, dropboxConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to configure dropbox") + } + + f := &Fs{ + name: name, + opt: *opt, + pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), + } + config := dropbox.Config{ + LogLevel: dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo + Client: oAuthClient, // maybe??? + HeaderGenerator: f.headerGenerator, + } + f.srv = files.New(config) + f.sharing = sharing.New(config) + f.users = users.New(config) + f.features = (&fs.Features{ + CaseInsensitive: true, + ReadMimeType: true, + CanHaveEmptyDirectories: true, + }).Fill(f) + f.setRoot(root) + + // If root starts with / then use the actual root + if strings.HasPrefix(root, "/") { + var acc *users.FullAccount + err = f.pacer.Call(func() (bool, error) { + acc, err = f.users.GetCurrentAccount() + return shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "get current account failed") + } + switch x := acc.RootInfo.(type) { + case *common.TeamRootInfo: + f.ns = x.RootNamespaceId + case *common.UserRootInfo: + f.ns = x.RootNamespaceId + default: + return nil, errors.Errorf("unknown RootInfo type %v %T", acc.RootInfo, acc.RootInfo) + } + fs.Debugf(f, "Using root namespace %q", f.ns) + } + + // See if the root is actually an object + _, err = f.getFileMetadata(f.slashRoot) + if err == nil { + newRoot := path.Dir(f.root) + if newRoot == "." { + newRoot = "" + } + f.setRoot(newRoot) + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + return f, nil +} + +// headerGenerator for dropbox sdk +func (f *Fs) headerGenerator(hostType string, style string, namespace string, route string) map[string]string { + if f.ns == "" { + return map[string]string{} + } + return map[string]string{ + "Dropbox-API-Path-Root": `{".tag": "namespace_id", "namespace_id": "` + f.ns + `"}`, + } +} + +// Sets root in f +func (f *Fs) setRoot(root string) { + f.root = strings.Trim(root, "/") + f.slashRoot = "/" + f.root + f.slashRootSlash = f.slashRoot + if f.root != "" { + f.slashRootSlash += "/" + } +} + +// getMetadata gets the metadata for a file or directory +func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) { + err = f.pacer.Call(func() (bool, error) { + entry, err = f.srv.GetMetadata(&files.GetMetadataArg{Path: objPath}) + return shouldRetry(err) + }) + if err != nil { + switch e := err.(type) { + case files.GetMetadataAPIError: + switch e.EndpointError.Path.Tag { + case files.LookupErrorNotFound: + notFound = true + err = nil + } + } + } + return +} + +// getFileMetadata gets the metadata for a file +func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err error) { + entry, notFound, err := f.getMetadata(filePath) + if err != nil { + return nil, err + } + if notFound { + return nil, fs.ErrorObjectNotFound + } + fileInfo, ok := entry.(*files.FileMetadata) + if !ok { + return nil, fs.ErrorNotAFile + } + return fileInfo, nil +} + +// getDirMetadata gets the metadata for a directory +func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err error) { + entry, notFound, err := f.getMetadata(dirPath) + if err != nil { + return nil, err + } + if notFound { + return nil, fs.ErrorDirNotFound + } + dirInfo, ok := entry.(*files.FolderMetadata) + if !ok { + return nil, fs.ErrorIsFile + } + return dirInfo, nil +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + err = o.setMetadataFromEntry(info) + } else { + err = o.readEntryAndSetMetadata() + } + if err != nil { + return nil, err + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + root := f.slashRoot + if dir != "" { + root += "/" + dir + } + + started := false + var res *files.ListFolderResult + for { + if !started { + arg := files.ListFolderArg{ + Path: root, + Recursive: false, + } + if root == "/" { + arg.Path = "" // Specify root folder as empty string + } + err = f.pacer.Call(func() (bool, error) { + res, err = f.srv.ListFolder(&arg) + return shouldRetry(err) + }) + if err != nil { + switch e := err.(type) { + case files.ListFolderAPIError: + switch e.EndpointError.Path.Tag { + case files.LookupErrorNotFound: + err = fs.ErrorDirNotFound + } + } + return nil, err + } + started = true + } else { + arg := files.ListFolderContinueArg{ + Cursor: res.Cursor, + } + err = f.pacer.Call(func() (bool, error) { + res, err = f.srv.ListFolderContinue(&arg) + return shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "list continue") + } + } + for _, entry := range res.Entries { + var fileInfo *files.FileMetadata + var folderInfo *files.FolderMetadata + var metadata *files.Metadata + switch info := entry.(type) { + case *files.FolderMetadata: + folderInfo = info + metadata = &info.Metadata + case *files.FileMetadata: + fileInfo = info + metadata = &info.Metadata + default: + fs.Errorf(f, "Unknown type %T", entry) + continue + } + + // Only the last element is reliably cased in PathDisplay + entryPath := metadata.PathDisplay + leaf := path.Base(entryPath) + remote := path.Join(dir, leaf) + if folderInfo != nil { + d := fs.NewDir(remote, time.Now()) + entries = append(entries, d) + } else if fileInfo != nil { + o, err := f.newObjectWithInfo(remote, fileInfo) + if err != nil { + return nil, err + } + entries = append(entries, o) + } + } + if !res.HasMore { + break + } + } + return entries, nil +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + // Temporary Object under construction + o := &Object{ + fs: f, + remote: src.Remote(), + } + return o, o.Update(in, src, options...) +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + root := path.Join(f.slashRoot, dir) + + // can't create or run metadata on root + if root == "/" { + return nil + } + + // check directory doesn't exist + _, err := f.getDirMetadata(root) + if err == nil { + return nil // directory exists already + } else if err != fs.ErrorDirNotFound { + return err // some other error + } + + // create it + arg2 := files.CreateFolderArg{ + Path: root, + } + err = f.pacer.Call(func() (bool, error) { + _, err = f.srv.CreateFolderV2(&arg2) + return shouldRetry(err) + }) + return err +} + +// Rmdir deletes the container +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + root := path.Join(f.slashRoot, dir) + + // can't remove root + if root == "/" { + return errors.New("can't remove root directory") + } + + // check directory exists + _, err := f.getDirMetadata(root) + if err != nil { + return errors.Wrap(err, "Rmdir") + } + + // check directory empty + arg := files.ListFolderArg{ + Path: root, + Recursive: false, + } + if root == "/" { + arg.Path = "" // Specify root folder as empty string + } + var res *files.ListFolderResult + err = f.pacer.Call(func() (bool, error) { + res, err = f.srv.ListFolder(&arg) + return shouldRetry(err) + }) + if err != nil { + return errors.Wrap(err, "Rmdir") + } + if len(res.Entries) != 0 { + return errors.New("directory not empty") + } + + // remove it + err = f.pacer.Call(func() (bool, error) { + _, err = f.srv.DeleteV2(&files.DeleteArg{Path: root}) + return shouldRetry(err) + }) + return err +} + +// Precision returns the precision +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + + // Temporary Object under construction + dstObj := &Object{ + fs: f, + remote: remote, + } + + // Copy + arg := files.RelocationArg{} + arg.FromPath = srcObj.remotePath() + arg.ToPath = dstObj.remotePath() + var err error + var result *files.RelocationResult + err = f.pacer.Call(func() (bool, error) { + result, err = f.srv.CopyV2(&arg) + return shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "copy failed") + } + + // Set the metadata + fileInfo, ok := result.Metadata.(*files.FileMetadata) + if !ok { + return nil, fs.ErrorNotAFile + } + err = dstObj.setMetadataFromEntry(fileInfo) + if err != nil { + return nil, errors.Wrap(err, "copy failed") + } + + return dstObj, nil +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() (err error) { + // Let dropbox delete the filesystem tree + err = f.pacer.Call(func() (bool, error) { + _, err = f.srv.DeleteV2(&files.DeleteArg{Path: f.slashRoot}) + return shouldRetry(err) + }) + return err +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // Temporary Object under construction + dstObj := &Object{ + fs: f, + remote: remote, + } + + // Do the move + arg := files.RelocationArg{} + arg.FromPath = srcObj.remotePath() + arg.ToPath = dstObj.remotePath() + var err error + var result *files.RelocationResult + err = f.pacer.Call(func() (bool, error) { + result, err = f.srv.MoveV2(&arg) + return shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "move failed") + } + + // Set the metadata + fileInfo, ok := result.Metadata.(*files.FileMetadata) + if !ok { + return nil, fs.ErrorNotAFile + } + err = dstObj.setMetadataFromEntry(fileInfo) + if err != nil { + return nil, errors.Wrap(err, "move failed") + } + return dstObj, nil +} + +// PublicLink adds a "readable by anyone with link" permission on the given file or folder. +func (f *Fs) PublicLink(remote string) (link string, err error) { + absPath := "/" + path.Join(f.Root(), remote) + fs.Debugf(f, "attempting to share '%s' (absolute path: %s)", remote, absPath) + createArg := sharing.CreateSharedLinkWithSettingsArg{ + Path: absPath, + } + var linkRes sharing.IsSharedLinkMetadata + err = f.pacer.Call(func() (bool, error) { + linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg) + return shouldRetry(err) + }) + + if err != nil && strings.Contains(err.Error(), sharing.CreateSharedLinkWithSettingsErrorSharedLinkAlreadyExists) { + fs.Debugf(absPath, "has a public link already, attempting to retrieve it") + listArg := sharing.ListSharedLinksArg{ + Path: absPath, + DirectOnly: true, + } + var listRes *sharing.ListSharedLinksResult + err = f.pacer.Call(func() (bool, error) { + listRes, err = f.sharing.ListSharedLinks(&listArg) + return shouldRetry(err) + }) + if err != nil { + return + } + if len(listRes.Links) == 0 { + err = errors.New("Dropbox says the sharing link already exists, but list came back empty") + return + } + linkRes = listRes.Links[0] + } + if err == nil { + switch res := linkRes.(type) { + case *sharing.FileLinkMetadata: + link = res.Url + case *sharing.FolderLinkMetadata: + link = res.Url + default: + err = fmt.Errorf("Don't know how to extract link, response has unknown format: %T", res) + } + } + return +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := path.Join(srcFs.slashRoot, srcRemote) + dstPath := path.Join(f.slashRoot, dstRemote) + + // Check if destination exists + _, err := f.getDirMetadata(dstPath) + if err == nil { + return fs.ErrorDirExists + } else if err != fs.ErrorDirNotFound { + return err + } + + // Make sure the parent directory exists + // ...apparently not necessary + + // Do the move + arg := files.RelocationArg{} + arg.FromPath = srcPath + arg.ToPath = dstPath + err = f.pacer.Call(func() (bool, error) { + _, err = f.srv.MoveV2(&arg) + return shouldRetry(err) + }) + if err != nil { + return errors.Wrap(err, "MoveDir failed") + } + + return nil +} + +// About gets quota information +func (f *Fs) About() (usage *fs.Usage, err error) { + var q *users.SpaceUsage + err = f.pacer.Call(func() (bool, error) { + q, err = f.users.GetSpaceUsage() + return shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "about failed") + } + var total uint64 + if q.Allocation != nil { + if q.Allocation.Individual != nil { + total += q.Allocation.Individual.Allocated + } + if q.Allocation.Team != nil { + total += q.Allocation.Team.Allocated + } + } + usage = &fs.Usage{ + Total: fs.NewUsageValue(int64(total)), // quota of bytes that can be used + Used: fs.NewUsageValue(int64(q.Used)), // bytes in use + Free: fs.NewUsageValue(int64(total - q.Used)), // bytes which can be uploaded before reaching the quota + } + return usage, nil +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.Dropbox) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the dropbox special hash +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.Dropbox { + return "", hash.ErrUnsupported + } + err := o.readMetaData() + if err != nil { + return "", errors.Wrap(err, "failed to read hash from metadata") + } + return o.hash, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.bytes +} + +// setMetadataFromEntry sets the fs data from a files.FileMetadata +// +// This isn't a complete set of metadata and has an inacurate date +func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error { + o.bytes = int64(info.Size) + o.modTime = info.ClientModified + o.hash = info.ContentHash + return nil +} + +// Reads the entry for a file from dropbox +func (o *Object) readEntry() (*files.FileMetadata, error) { + return o.fs.getFileMetadata(o.remotePath()) +} + +// Read entry if not set and set metadata from it +func (o *Object) readEntryAndSetMetadata() error { + // Last resort set time from client + if !o.modTime.IsZero() { + return nil + } + entry, err := o.readEntry() + if err != nil { + return err + } + return o.setMetadataFromEntry(entry) +} + +// Returns the remote path for the object +func (o *Object) remotePath() string { + return o.fs.slashRootSlash + o.remote +} + +// readMetaData gets the info if it hasn't already been fetched +func (o *Object) readMetaData() (err error) { + if !o.modTime.IsZero() { + return nil + } + // Last resort + return o.readEntryAndSetMetadata() +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Debugf(o, "Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// SetModTime sets the modification time of the local fs object +// +// Commits the datastore +func (o *Object) SetModTime(modTime time.Time) error { + // Dropbox doesn't have a way of doing this so returning this + // error will cause the file to be deleted first then + // re-uploaded to set the time. + return fs.ErrorCantSetModTimeWithoutDelete +} + +// Storable returns whether this object is storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + headers := fs.OpenOptionHeaders(options) + arg := files.DownloadArg{Path: o.remotePath(), ExtraHeaders: headers} + err = o.fs.pacer.Call(func() (bool, error) { + _, in, err = o.fs.srv.Download(&arg) + return shouldRetry(err) + }) + + switch e := err.(type) { + case files.DownloadAPIError: + // Don't attempt to retry copyright violation errors + if e.EndpointError.Path.Tag == files.LookupErrorRestrictedContent { + return nil, fserrors.NoRetryError(err) + } + } + + return +} + +// uploadChunked uploads the object in parts +// +// Will work optimally if size is >= uploadChunkSize. If the size is either +// unknown (i.e. -1) or smaller than uploadChunkSize, the method incurs an +// avoidable request to the Dropbox API that does not carry payload. +func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) { + chunkSize := int64(o.fs.opt.ChunkSize) + chunks := 0 + if size != -1 { + chunks = int(size/chunkSize) + 1 + } + in := readers.NewCountingReader(in0) + buf := make([]byte, int(chunkSize)) + + fmtChunk := func(cur int, last bool) { + if chunks == 0 && last { + fs.Debugf(o, "Streaming chunk %d/%d", cur, cur) + } else if chunks == 0 { + fs.Debugf(o, "Streaming chunk %d/unknown", cur) + } else { + fs.Debugf(o, "Uploading chunk %d/%d", cur, chunks) + } + } + + // write the first chunk + fmtChunk(1, false) + var res *files.UploadSessionStartResult + chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize) + err = o.fs.pacer.Call(func() (bool, error) { + // seek to the start in case this is a retry + if _, err = chunk.Seek(0, io.SeekStart); err != nil { + return false, nil + } + res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, chunk) + return shouldRetry(err) + }) + if err != nil { + return nil, err + } + + cursor := files.UploadSessionCursor{ + SessionId: res.SessionId, + Offset: 0, + } + appendArg := files.UploadSessionAppendArg{ + Cursor: &cursor, + Close: false, + } + + // write more whole chunks (if any) + currentChunk := 2 + for { + if chunks > 0 && currentChunk >= chunks { + // if the size is known, only upload full chunks. Remaining bytes are uploaded with + // the UploadSessionFinish request. + break + } else if chunks == 0 && in.BytesRead()-cursor.Offset < uint64(chunkSize) { + // if the size is unknown, upload as long as we can read full chunks from the reader. + // The UploadSessionFinish request will not contain any payload. + break + } + cursor.Offset = in.BytesRead() + fmtChunk(currentChunk, false) + chunk = readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize) + err = o.fs.pacer.Call(func() (bool, error) { + // seek to the start in case this is a retry + if _, err = chunk.Seek(0, io.SeekStart); err != nil { + return false, nil + } + err = o.fs.srv.UploadSessionAppendV2(&appendArg, chunk) + // after the first chunk is uploaded, we retry everything + return err != nil, err + }) + if err != nil { + return nil, err + } + currentChunk++ + } + + // write the remains + cursor.Offset = in.BytesRead() + args := &files.UploadSessionFinishArg{ + Cursor: &cursor, + Commit: commitInfo, + } + fmtChunk(currentChunk, true) + chunk = readers.NewRepeatableReaderBuffer(in, buf) + err = o.fs.pacer.Call(func() (bool, error) { + // seek to the start in case this is a retry + if _, err = chunk.Seek(0, io.SeekStart); err != nil { + return false, nil + } + entry, err = o.fs.srv.UploadSessionFinish(args, chunk) + // after the first chunk is uploaded, we retry everything + return err != nil, err + }) + if err != nil { + return nil, err + } + return entry, nil +} + +// Update the already existing object +// +// Copy the reader into the object updating modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + remote := o.remotePath() + if ignoredFiles.MatchString(remote) { + fs.Logf(o, "File name disallowed - not uploading") + return nil + } + commitInfo := files.NewCommitInfo(o.remotePath()) + commitInfo.Mode.Tag = "overwrite" + // The Dropbox API only accepts timestamps in UTC with second precision. + commitInfo.ClientModified = src.ModTime().UTC().Round(time.Second) + + size := src.Size() + var err error + var entry *files.FileMetadata + if size > int64(o.fs.opt.ChunkSize) || size == -1 { + entry, err = o.uploadChunked(in, commitInfo, size) + } else { + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + entry, err = o.fs.srv.Upload(commitInfo, in) + return shouldRetry(err) + }) + } + if err != nil { + return errors.Wrap(err, "upload failed") + } + return o.setMetadataFromEntry(entry) +} + +// Remove an object +func (o *Object) Remove() (err error) { + err = o.fs.pacer.Call(func() (bool, error) { + _, err = o.fs.srv.DeleteV2(&files.DeleteArg{Path: o.remotePath()}) + return shouldRetry(err) + }) + return err +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.PutStreamer = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.PublicLinker = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) + _ fs.Object = (*Object)(nil) +) diff --git a/.rclone_repo/backend/dropbox/dropbox_test.go b/.rclone_repo/backend/dropbox/dropbox_test.go new file mode 100755 index 0000000..e4f9bb6 --- /dev/null +++ b/.rclone_repo/backend/dropbox/dropbox_test.go @@ -0,0 +1,17 @@ +// Test Dropbox filesystem interface +package dropbox_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/dropbox" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestDropbox:", + NilObject: (*dropbox.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/ftp/ftp.go b/.rclone_repo/backend/ftp/ftp.go new file mode 100755 index 0000000..21b7f5e --- /dev/null +++ b/.rclone_repo/backend/ftp/ftp.go @@ -0,0 +1,761 @@ +// Package ftp interfaces with FTP servers +package ftp + +import ( + "io" + "net/textproto" + "os" + "path" + "sync" + "time" + + "github.com/jlaffaye/ftp" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/readers" + "github.com/pkg/errors" +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "ftp", + Description: "FTP Connection", + NewFs: NewFs, + Options: []fs.Option{ + { + Name: "host", + Help: "FTP host to connect to", + Required: true, + Examples: []fs.OptionExample{{ + Value: "ftp.example.com", + Help: "Connect to ftp.example.com", + }}, + }, { + Name: "user", + Help: "FTP username, leave blank for current username, " + os.Getenv("USER"), + }, { + Name: "port", + Help: "FTP port, leave blank to use default (21)", + }, { + Name: "pass", + Help: "FTP password", + IsPassword: true, + Required: true, + }, + }, + }) +} + +// Options defines the configuration for this backend +type Options struct { + Host string `config:"host"` + User string `config:"user"` + Pass string `config:"pass"` + Port string `config:"port"` +} + +// Fs represents a remote FTP server +type Fs struct { + name string // name of this remote + root string // the path we are working on if any + opt Options // parsed options + features *fs.Features // optional features + url string + user string + pass string + dialAddr string + poolMu sync.Mutex + pool []*ftp.ServerConn +} + +// Object describes an FTP file +type Object struct { + fs *Fs + remote string + info *FileInfo +} + +// FileInfo is the metadata known about an FTP file +type FileInfo struct { + Name string + Size uint64 + ModTime time.Time + IsDir bool +} + +// ------------------------------------------------------------ + +// Name of this fs +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String returns a description of the FS +func (f *Fs) String() string { + return f.url +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Open a new connection to the FTP server. +func (f *Fs) ftpConnection() (*ftp.ServerConn, error) { + fs.Debugf(f, "Connecting to FTP server") + c, err := ftp.DialTimeout(f.dialAddr, fs.Config.ConnectTimeout) + if err != nil { + fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err) + return nil, errors.Wrap(err, "ftpConnection Dial") + } + err = c.Login(f.user, f.pass) + if err != nil { + _ = c.Quit() + fs.Errorf(f, "Error while Logging in into %s: %s", f.dialAddr, err) + return nil, errors.Wrap(err, "ftpConnection Login") + } + return c, nil +} + +// Get an FTP connection from the pool, or open a new one +func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) { + f.poolMu.Lock() + if len(f.pool) > 0 { + c = f.pool[0] + f.pool = f.pool[1:] + } + f.poolMu.Unlock() + if c != nil { + return c, nil + } + return f.ftpConnection() +} + +// Return an FTP connection to the pool +// +// It nils the pointed to connection out so it can't be reused +// +// if err is not nil then it checks the connection is alive using a +// NOOP request +func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) { + c := *pc + *pc = nil + if err != nil { + // If not a regular FTP error code then check the connection + _, isRegularError := errors.Cause(err).(*textproto.Error) + if !isRegularError { + nopErr := c.NoOp() + if nopErr != nil { + fs.Debugf(f, "Connection failed, closing: %v", nopErr) + _ = c.Quit() + return + } + } + } + f.poolMu.Lock() + f.pool = append(f.pool, c) + f.poolMu.Unlock() +} + +// NewFs contstructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) { + // defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err) + // Parse config into Options struct + opt := new(Options) + err = configstruct.Set(m, opt) + if err != nil { + return nil, err + } + pass, err := obscure.Reveal(opt.Pass) + if err != nil { + return nil, errors.Wrap(err, "NewFS decrypt password") + } + user := opt.User + if user == "" { + user = os.Getenv("USER") + } + port := opt.Port + if port == "" { + port = "21" + } + + dialAddr := opt.Host + ":" + port + u := "ftp://" + path.Join(dialAddr+"/", root) + f := &Fs{ + name: name, + root: root, + opt: *opt, + url: u, + user: user, + pass: pass, + dialAddr: dialAddr, + } + f.features = (&fs.Features{ + CanHaveEmptyDirectories: true, + }).Fill(f) + // Make a connection and pool it to return errors early + c, err := f.getFtpConnection() + if err != nil { + return nil, errors.Wrap(err, "NewFs") + } + f.putFtpConnection(&c, nil) + if root != "" { + // Check to see if the root actually an existing file + remote := path.Base(root) + f.root = path.Dir(root) + if f.root == "." { + f.root = "" + } + _, err := f.NewObject(remote) + if err != nil { + if err == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile { + // File doesn't exist so return old f + f.root = root + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + return f, err +} + +// translateErrorFile turns FTP errors into rclone errors if possible for a file +func translateErrorFile(err error) error { + switch errX := err.(type) { + case *textproto.Error: + switch errX.Code { + case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored: + err = fs.ErrorObjectNotFound + } + } + return err +} + +// translateErrorDir turns FTP errors into rclone errors if possible for a directory +func translateErrorDir(err error) error { + switch errX := err.(type) { + case *textproto.Error: + switch errX.Code { + case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored: + err = fs.ErrorDirNotFound + } + } + return err +} + +// findItem finds a directory entry for the name in its parent directory +func (f *Fs) findItem(remote string) (entry *ftp.Entry, err error) { + // defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err) + fullPath := path.Join(f.root, remote) + dir := path.Dir(fullPath) + base := path.Base(fullPath) + + c, err := f.getFtpConnection() + if err != nil { + return nil, errors.Wrap(err, "findItem") + } + files, err := c.List(dir) + f.putFtpConnection(&c, err) + if err != nil { + return nil, translateErrorFile(err) + } + for _, file := range files { + if file.Name == base { + return file, nil + } + } + return nil, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (o fs.Object, err error) { + // defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err) + entry, err := f.findItem(remote) + if err != nil { + return nil, err + } + if entry != nil && entry.Type != ftp.EntryTypeFolder { + o := &Object{ + fs: f, + remote: remote, + } + info := &FileInfo{ + Name: remote, + Size: entry.Size, + ModTime: entry.Time, + } + o.info = info + + return o, nil + } + return nil, fs.ErrorObjectNotFound +} + +// dirExists checks the directory pointed to by remote exists or not +func (f *Fs) dirExists(remote string) (exists bool, err error) { + entry, err := f.findItem(remote) + if err != nil { + return false, errors.Wrap(err, "dirExists") + } + if entry != nil && entry.Type == ftp.EntryTypeFolder { + return true, nil + } + return false, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + // defer fs.Trace(dir, "curlevel=%d", curlevel)("") + c, err := f.getFtpConnection() + if err != nil { + return nil, errors.Wrap(err, "list") + } + files, err := c.List(path.Join(f.root, dir)) + f.putFtpConnection(&c, err) + if err != nil { + return nil, translateErrorDir(err) + } + // Annoyingly FTP returns success for a directory which + // doesn't exist, so check it really doesn't exist if no + // entries found. + if len(files) == 0 { + exists, err := f.dirExists(dir) + if err != nil { + return nil, errors.Wrap(err, "list") + } + if !exists { + return nil, fs.ErrorDirNotFound + } + } + for i := range files { + object := files[i] + newremote := path.Join(dir, object.Name) + switch object.Type { + case ftp.EntryTypeFolder: + if object.Name == "." || object.Name == ".." { + continue + } + d := fs.NewDir(newremote, object.Time) + entries = append(entries, d) + default: + o := &Object{ + fs: f, + remote: newremote, + } + info := &FileInfo{ + Name: newremote, + Size: object.Size, + ModTime: object.Time, + } + o.info = info + entries = append(entries, o) + } + } + return entries, nil +} + +// Hashes are not supported +func (f *Fs) Hashes() hash.Set { + return 0 +} + +// Precision shows Modified Time not supported +func (f *Fs) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +// Put in to the remote path with the modTime given of the given size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + // fs.Debugf(f, "Trying to put file %s", src.Remote()) + err := f.mkParentDir(src.Remote()) + if err != nil { + return nil, errors.Wrap(err, "Put mkParentDir failed") + } + o := &Object{ + fs: f, + remote: src.Remote(), + } + err = o.Update(in, src, options...) + return o, err +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// getInfo reads the FileInfo for a path +func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) { + // defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err) + dir := path.Dir(remote) + base := path.Base(remote) + + c, err := f.getFtpConnection() + if err != nil { + return nil, errors.Wrap(err, "getInfo") + } + files, err := c.List(dir) + f.putFtpConnection(&c, err) + if err != nil { + return nil, translateErrorFile(err) + } + + for i := range files { + if files[i].Name == base { + info := &FileInfo{ + Name: remote, + Size: files[i].Size, + ModTime: files[i].Time, + IsDir: files[i].Type == ftp.EntryTypeFolder, + } + return info, nil + } + } + return nil, fs.ErrorObjectNotFound +} + +// mkdir makes the directory and parents using unrooted paths +func (f *Fs) mkdir(abspath string) error { + if abspath == "." || abspath == "/" { + return nil + } + fi, err := f.getInfo(abspath) + if err == nil { + if fi.IsDir { + return nil + } + return fs.ErrorIsFile + } else if err != fs.ErrorObjectNotFound { + return errors.Wrapf(err, "mkdir %q failed", abspath) + } + parent := path.Dir(abspath) + err = f.mkdir(parent) + if err != nil { + return err + } + c, connErr := f.getFtpConnection() + if connErr != nil { + return errors.Wrap(connErr, "mkdir") + } + err = c.MakeDir(abspath) + f.putFtpConnection(&c, err) + switch errX := err.(type) { + case *textproto.Error: + switch errX.Code { + case ftp.StatusFileUnavailable: // dir already exists: see issue #2181 + err = nil + case 521: // dir already exists: error number according to RFC 959: issue #2363 + err = nil + } + } + return err +} + +// mkParentDir makes the parent of remote if necessary and any +// directories above that +func (f *Fs) mkParentDir(remote string) error { + parent := path.Dir(remote) + return f.mkdir(path.Join(f.root, parent)) +} + +// Mkdir creates the directory if it doesn't exist +func (f *Fs) Mkdir(dir string) (err error) { + // defer fs.Trace(dir, "")("err=%v", &err) + root := path.Join(f.root, dir) + return f.mkdir(root) +} + +// Rmdir removes the directory (container, bucket) if empty +// +// Return an error if it doesn't exist or isn't empty +func (f *Fs) Rmdir(dir string) error { + c, err := f.getFtpConnection() + if err != nil { + return errors.Wrap(translateErrorFile(err), "Rmdir") + } + err = c.RemoveDir(path.Join(f.root, dir)) + f.putFtpConnection(&c, err) + return translateErrorDir(err) +} + +// Move renames a remote file object +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + err := f.mkParentDir(remote) + if err != nil { + return nil, errors.Wrap(err, "Move mkParentDir failed") + } + c, err := f.getFtpConnection() + if err != nil { + return nil, errors.Wrap(err, "Move") + } + err = c.Rename( + path.Join(srcObj.fs.root, srcObj.remote), + path.Join(f.root, remote), + ) + f.putFtpConnection(&c, err) + if err != nil { + return nil, errors.Wrap(err, "Move Rename failed") + } + dstObj, err := f.NewObject(remote) + if err != nil { + return nil, errors.Wrap(err, "Move NewObject failed") + } + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := path.Join(srcFs.root, srcRemote) + dstPath := path.Join(f.root, dstRemote) + + // Check if destination exists + fi, err := f.getInfo(dstPath) + if err == nil { + if fi.IsDir { + return fs.ErrorDirExists + } + return fs.ErrorIsFile + } else if err != fs.ErrorObjectNotFound { + return errors.Wrapf(err, "DirMove getInfo failed") + } + + // Make sure the parent directory exists + err = f.mkdir(path.Dir(dstPath)) + if err != nil { + return errors.Wrap(err, "DirMove mkParentDir dst failed") + } + + // Do the move + c, err := f.getFtpConnection() + if err != nil { + return errors.Wrap(err, "DirMove") + } + err = c.Rename( + srcPath, + dstPath, + ) + f.putFtpConnection(&c, err) + if err != nil { + return errors.Wrapf(err, "DirMove Rename(%q,%q) failed", srcPath, dstPath) + } + return nil +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// String version of o +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the hash of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return int64(o.info.Size) +} + +// ModTime returns the modification time of the object +func (o *Object) ModTime() time.Time { + return o.info.ModTime +} + +// SetModTime sets the modification time of the object +func (o *Object) SetModTime(modTime time.Time) error { + return nil +} + +// Storable returns a boolean as to whether this object is storable +func (o *Object) Storable() bool { + return true +} + +// ftpReadCloser implements io.ReadCloser for FTP objects. +type ftpReadCloser struct { + rc io.ReadCloser + c *ftp.ServerConn + f *Fs + err error // errors found during read +} + +// Read bytes into p +func (f *ftpReadCloser) Read(p []byte) (n int, err error) { + n, err = f.rc.Read(p) + if err != nil && err != io.EOF { + f.err = err // store any errors for Close to examine + } + return +} + +// Close the FTP reader and return the connection to the pool +func (f *ftpReadCloser) Close() error { + err := f.rc.Close() + // if errors while reading or closing, dump the connection + if err != nil || f.err != nil { + _ = f.c.Quit() + } else { + f.f.putFtpConnection(&f.c, nil) + } + // mask the error if it was caused by a premature close + switch errX := err.(type) { + case *textproto.Error: + switch errX.Code { + case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable: + err = nil + } + } + return err +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) { + // defer fs.Trace(o, "")("rc=%v, err=%v", &rc, &err) + path := path.Join(o.fs.root, o.remote) + var offset, limit int64 = 0, -1 + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + case *fs.RangeOption: + offset, limit = x.Decode(o.Size()) + default: + if option.Mandatory() { + fs.Logf(o, "Unsupported mandatory option: %v", option) + } + } + } + c, err := o.fs.getFtpConnection() + if err != nil { + return nil, errors.Wrap(err, "open") + } + fd, err := c.RetrFrom(path, uint64(offset)) + if err != nil { + o.fs.putFtpConnection(&c, err) + return nil, errors.Wrap(err, "open") + } + rc = &ftpReadCloser{rc: readers.NewLimitedReadCloser(fd, limit), c: c, f: o.fs} + return rc, nil +} + +// Update the already existing object +// +// Copy the reader into the object updating modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + // defer fs.Trace(o, "src=%v", src)("err=%v", &err) + path := path.Join(o.fs.root, o.remote) + // remove the file if upload failed + remove := func() { + removeErr := o.Remove() + if removeErr != nil { + fs.Debugf(o, "Failed to remove: %v", removeErr) + } else { + fs.Debugf(o, "Removed after failed upload: %v", err) + } + } + c, err := o.fs.getFtpConnection() + if err != nil { + return errors.Wrap(err, "Update") + } + err = c.Stor(path, in) + if err != nil { + _ = c.Quit() + remove() + return errors.Wrap(err, "update stor") + } + o.fs.putFtpConnection(&c, nil) + o.info, err = o.fs.getInfo(path) + if err != nil { + return errors.Wrap(err, "update getinfo") + } + return nil +} + +// Remove an object +func (o *Object) Remove() (err error) { + // defer fs.Trace(o, "")("err=%v", &err) + path := path.Join(o.fs.root, o.remote) + // Check if it's a directory or a file + info, err := o.fs.getInfo(path) + if err != nil { + return err + } + if info.IsDir { + err = o.fs.Rmdir(o.remote) + } else { + c, err := o.fs.getFtpConnection() + if err != nil { + return errors.Wrap(err, "Remove") + } + err = c.Delete(path) + o.fs.putFtpConnection(&c, err) + } + return err +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.Mover = &Fs{} + _ fs.DirMover = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.Object = &Object{} +) diff --git a/.rclone_repo/backend/ftp/ftp_test.go b/.rclone_repo/backend/ftp/ftp_test.go new file mode 100755 index 0000000..bb71e73 --- /dev/null +++ b/.rclone_repo/backend/ftp/ftp_test.go @@ -0,0 +1,17 @@ +// Test FTP filesystem interface +package ftp_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/ftp" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestFTP:", + NilObject: (*ftp.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/googlecloudstorage/googlecloudstorage.go b/.rclone_repo/backend/googlecloudstorage/googlecloudstorage.go new file mode 100755 index 0000000..1d74bcc --- /dev/null +++ b/.rclone_repo/backend/googlecloudstorage/googlecloudstorage.go @@ -0,0 +1,991 @@ +// Package googlecloudstorage provides an interface to Google Cloud Storage +package googlecloudstorage + +/* +Notes + +Can't set Updated but can set Metadata on object creation + +Patch needs full_control not just read_write + +FIXME Patch/Delete/Get isn't working with files with spaces in - giving 404 error +- https://code.google.com/p/google-api-go-client/issues/detail?id=64 +*/ + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path" + "regexp" + "strings" + "sync" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/fs/walk" + "github.com/ncw/rclone/lib/oauthutil" + "github.com/ncw/rclone/lib/pacer" + "github.com/pkg/errors" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/api/googleapi" + storage "google.golang.org/api/storage/v1" +) + +const ( + rcloneClientID = "202264815644.apps.googleusercontent.com" + rcloneEncryptedClientSecret = "Uj7C9jGfb9gmeaV70Lh058cNkWvepr-Es9sBm0zdgil7JaOWF1VySw" + timeFormatIn = time.RFC3339 + timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00" + metaMtime = "mtime" // key to store mtime under in metadata + listChunks = 1000 // chunk size to read directory listings + minSleep = 10 * time.Millisecond +) + +var ( + // Description of how to auth for this app + storageConfig = &oauth2.Config{ + Scopes: []string{storage.DevstorageFullControlScope}, + Endpoint: google.Endpoint, + ClientID: rcloneClientID, + ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), + RedirectURL: oauthutil.TitleBarRedirectURL, + } +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "google cloud storage", + Prefix: "gcs", + Description: "Google Cloud Storage (this is not Google Drive)", + NewFs: NewFs, + Config: func(name string, m configmap.Mapper) { + saFile, _ := m.Get("service_account_file") + saCreds, _ := m.Get("service_account_credentials") + if saFile != "" || saCreds != "" { + return + } + err := oauthutil.Config("google cloud storage", name, m, storageConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + }, + Options: []fs.Option{{ + Name: config.ConfigClientID, + Help: "Google Application Client Id\nLeave blank normally.", + }, { + Name: config.ConfigClientSecret, + Help: "Google Application Client Secret\nLeave blank normally.", + }, { + Name: "project_number", + Help: "Project number.\nOptional - needed only for list/create/delete buckets - see your developer console.", + }, { + Name: "service_account_file", + Help: "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.", + }, { + Name: "service_account_credentials", + Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.", + Hide: fs.OptionHideBoth, + }, { + Name: "object_acl", + Help: "Access Control List for new objects.", + Examples: []fs.OptionExample{{ + Value: "authenticatedRead", + Help: "Object owner gets OWNER access, and all Authenticated Users get READER access.", + }, { + Value: "bucketOwnerFullControl", + Help: "Object owner gets OWNER access, and project team owners get OWNER access.", + }, { + Value: "bucketOwnerRead", + Help: "Object owner gets OWNER access, and project team owners get READER access.", + }, { + Value: "private", + Help: "Object owner gets OWNER access [default if left blank].", + }, { + Value: "projectPrivate", + Help: "Object owner gets OWNER access, and project team members get access according to their roles.", + }, { + Value: "publicRead", + Help: "Object owner gets OWNER access, and all Users get READER access.", + }}, + }, { + Name: "bucket_acl", + Help: "Access Control List for new buckets.", + Examples: []fs.OptionExample{{ + Value: "authenticatedRead", + Help: "Project team owners get OWNER access, and all Authenticated Users get READER access.", + }, { + Value: "private", + Help: "Project team owners get OWNER access [default if left blank].", + }, { + Value: "projectPrivate", + Help: "Project team members get access according to their roles.", + }, { + Value: "publicRead", + Help: "Project team owners get OWNER access, and all Users get READER access.", + }, { + Value: "publicReadWrite", + Help: "Project team owners get OWNER access, and all Users get WRITER access.", + }}, + }, { + Name: "location", + Help: "Location for the newly created buckets.", + Examples: []fs.OptionExample{{ + Value: "", + Help: "Empty for default location (US).", + }, { + Value: "asia", + Help: "Multi-regional location for Asia.", + }, { + Value: "eu", + Help: "Multi-regional location for Europe.", + }, { + Value: "us", + Help: "Multi-regional location for United States.", + }, { + Value: "asia-east1", + Help: "Taiwan.", + }, { + Value: "asia-northeast1", + Help: "Tokyo.", + }, { + Value: "asia-southeast1", + Help: "Singapore.", + }, { + Value: "australia-southeast1", + Help: "Sydney.", + }, { + Value: "europe-west1", + Help: "Belgium.", + }, { + Value: "europe-west2", + Help: "London.", + }, { + Value: "us-central1", + Help: "Iowa.", + }, { + Value: "us-east1", + Help: "South Carolina.", + }, { + Value: "us-east4", + Help: "Northern Virginia.", + }, { + Value: "us-west1", + Help: "Oregon.", + }}, + }, { + Name: "storage_class", + Help: "The storage class to use when storing objects in Google Cloud Storage.", + Examples: []fs.OptionExample{{ + Value: "", + Help: "Default", + }, { + Value: "MULTI_REGIONAL", + Help: "Multi-regional storage class", + }, { + Value: "REGIONAL", + Help: "Regional storage class", + }, { + Value: "NEARLINE", + Help: "Nearline storage class", + }, { + Value: "COLDLINE", + Help: "Coldline storage class", + }, { + Value: "DURABLE_REDUCED_AVAILABILITY", + Help: "Durable reduced availability storage class", + }}, + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + ProjectNumber string `config:"project_number"` + ServiceAccountFile string `config:"service_account_file"` + ServiceAccountCredentials string `config:"service_account_credentials"` + ObjectACL string `config:"object_acl"` + BucketACL string `config:"bucket_acl"` + Location string `config:"location"` + StorageClass string `config:"storage_class"` +} + +// Fs represents a remote storage server +type Fs struct { + name string // name of this remote + root string // the path we are working on if any + opt Options // parsed options + features *fs.Features // optional features + svc *storage.Service // the connection to the storage server + client *http.Client // authorized client + bucket string // the bucket we are working on + bucketOKMu sync.Mutex // mutex to protect bucket OK + bucketOK bool // true if we have created the bucket + pacer *pacer.Pacer // To pace the API calls +} + +// Object describes a storage object +// +// Will definitely have info but maybe not meta +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + url string // download path + md5sum string // The MD5Sum of the object + bytes int64 // Bytes in the object + modTime time.Time // Modified time of the object + mimeType string +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + if f.root == "" { + return f.bucket + } + return f.bucket + "/" + f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + if f.root == "" { + return fmt.Sprintf("Storage bucket %s", f.bucket) + } + return fmt.Sprintf("Storage bucket %s path %s", f.bucket, f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// shouldRetry determines whehter a given err rates being retried +func shouldRetry(err error) (again bool, errOut error) { + again = false + if err != nil { + if fserrors.ShouldRetry(err) { + again = true + } else { + switch gerr := err.(type) { + case *googleapi.Error: + if gerr.Code >= 500 && gerr.Code < 600 { + // All 5xx errors should be retried + again = true + } else if len(gerr.Errors) > 0 { + reason := gerr.Errors[0].Reason + if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" { + again = true + } + } + } + } + } + return again, err +} + +// Pattern to match a storage path +var matcher = regexp.MustCompile(`^([^/]*)(.*)$`) + +// parseParse parses a storage 'url' +func parsePath(path string) (bucket, directory string, err error) { + parts := matcher.FindStringSubmatch(path) + if parts == nil { + err = errors.Errorf("couldn't find bucket in storage path %q", path) + } else { + bucket, directory = parts[1], parts[2] + directory = strings.Trim(directory, "/") + } + return +} + +func getServiceAccountClient(credentialsData []byte) (*http.Client, error) { + conf, err := google.JWTConfigFromJSON(credentialsData, storageConfig.Scopes...) + if err != nil { + return nil, errors.Wrap(err, "error processing credentials") + } + ctxWithSpecialClient := oauthutil.Context(fshttp.NewClient(fs.Config)) + return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil +} + +// NewFs contstructs an Fs from the path, bucket:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + var oAuthClient *http.Client + + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + if opt.ObjectACL == "" { + opt.ObjectACL = "private" + } + if opt.BucketACL == "" { + opt.BucketACL = "private" + } + + // try loading service account credentials from env variable, then from a file + if opt.ServiceAccountCredentials != "" && opt.ServiceAccountFile != "" { + loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile)) + if err != nil { + return nil, errors.Wrap(err, "error opening service account credentials file") + } + opt.ServiceAccountCredentials = string(loadedCreds) + } + if opt.ServiceAccountCredentials != "" { + oAuthClient, err = getServiceAccountClient([]byte(opt.ServiceAccountCredentials)) + if err != nil { + return nil, errors.Wrap(err, "failed configuring Google Cloud Storage Service Account") + } + } else { + oAuthClient, _, err = oauthutil.NewClient(name, m, storageConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to configure Google Cloud Storage") + } + } + + bucket, directory, err := parsePath(root) + if err != nil { + return nil, err + } + + f := &Fs{ + name: name, + bucket: bucket, + root: directory, + opt: *opt, + pacer: pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer), + } + f.features = (&fs.Features{ + ReadMimeType: true, + WriteMimeType: true, + BucketBased: true, + }).Fill(f) + + // Create a new authorized Drive client. + f.client = oAuthClient + f.svc, err = storage.New(f.client) + if err != nil { + return nil, errors.Wrap(err, "couldn't create Google Cloud Storage client") + } + + if f.root != "" { + f.root += "/" + // Check to see if the object exists + err = f.pacer.Call(func() (bool, error) { + _, err = f.svc.Objects.Get(bucket, directory).Do() + return shouldRetry(err) + }) + if err == nil { + f.root = path.Dir(directory) + if f.root == "." { + f.root = "" + } else { + f.root += "/" + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + } + return f, nil +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *storage.Object) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + if info != nil { + o.setMetaData(info) + } else { + err := o.readMetaData() // reads info and meta, returning an error + if err != nil { + return nil, err + } + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// listFn is called from list to handle an object. +type listFn func(remote string, object *storage.Object, isDirectory bool) error + +// list the objects into the function supplied +// +// dir is the starting directory, "" for root +// +// Set recurse to read sub directories +func (f *Fs) list(dir string, recurse bool, fn listFn) (err error) { + root := f.root + rootLength := len(root) + if dir != "" { + root += dir + "/" + } + list := f.svc.Objects.List(f.bucket).Prefix(root).MaxResults(listChunks) + if !recurse { + list = list.Delimiter("/") + } + for { + var objects *storage.Objects + err = f.pacer.Call(func() (bool, error) { + objects, err = list.Do() + return shouldRetry(err) + }) + if err != nil { + if gErr, ok := err.(*googleapi.Error); ok { + if gErr.Code == http.StatusNotFound { + err = fs.ErrorDirNotFound + } + } + return err + } + if !recurse { + var object storage.Object + for _, prefix := range objects.Prefixes { + if !strings.HasSuffix(prefix, "/") { + continue + } + err = fn(prefix[rootLength:len(prefix)-1], &object, true) + if err != nil { + return err + } + } + } + for _, object := range objects.Items { + if !strings.HasPrefix(object.Name, root) { + fs.Logf(f, "Odd name received %q", object.Name) + continue + } + remote := object.Name[rootLength:] + // is this a directory marker? + if (strings.HasSuffix(remote, "/") || remote == "") && object.Size == 0 { + if recurse && remote != "" { + // add a directory in if --fast-list since will have no prefixes + err = fn(remote[:len(remote)-1], object, true) + if err != nil { + return err + } + } + continue // skip directory marker + } + err = fn(remote, object, false) + if err != nil { + return err + } + } + if objects.NextPageToken == "" { + break + } + list.PageToken(objects.NextPageToken) + } + return nil +} + +// Convert a list item into a DirEntry +func (f *Fs) itemToDirEntry(remote string, object *storage.Object, isDirectory bool) (fs.DirEntry, error) { + if isDirectory { + d := fs.NewDir(remote, time.Time{}).SetSize(int64(object.Size)) + return d, nil + } + o, err := f.newObjectWithInfo(remote, object) + if err != nil { + return nil, err + } + return o, nil +} + +// mark the bucket as being OK +func (f *Fs) markBucketOK() { + if f.bucket != "" { + f.bucketOKMu.Lock() + f.bucketOK = true + f.bucketOKMu.Unlock() + } +} + +// listDir lists a single directory +func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { + // List the objects + err = f.list(dir, false, func(remote string, object *storage.Object, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + if entry != nil { + entries = append(entries, entry) + } + return nil + }) + if err != nil { + return nil, err + } + // bucket must be present if listing succeeded + f.markBucketOK() + return entries, err +} + +// listBuckets lists the buckets +func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) { + if dir != "" { + return nil, fs.ErrorListBucketRequired + } + if f.opt.ProjectNumber == "" { + return nil, errors.New("can't list buckets without project number") + } + listBuckets := f.svc.Buckets.List(f.opt.ProjectNumber).MaxResults(listChunks) + for { + var buckets *storage.Buckets + err = f.pacer.Call(func() (bool, error) { + buckets, err = listBuckets.Do() + return shouldRetry(err) + }) + if err != nil { + return nil, err + } + for _, bucket := range buckets.Items { + d := fs.NewDir(bucket.Name, time.Time{}) + entries = append(entries, d) + } + if buckets.NextPageToken == "" { + break + } + listBuckets.PageToken(buckets.NextPageToken) + } + return entries, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + if f.bucket == "" { + return f.listBuckets(dir) + } + return f.listDir(dir) +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + if f.bucket == "" { + return fs.ErrorListBucketRequired + } + list := walk.NewListRHelper(callback) + err = f.list(dir, true, func(remote string, object *storage.Object, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + return list.Add(entry) + }) + if err != nil { + return err + } + // bucket must be present if listing succeeded + f.markBucketOK() + return list.Flush() +} + +// Put the object into the bucket +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + // Temporary Object under construction + o := &Object{ + fs: f, + remote: src.Remote(), + } + return o, o.Update(in, src, options...) +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// Mkdir creates the bucket if it doesn't exist +func (f *Fs) Mkdir(dir string) (err error) { + f.bucketOKMu.Lock() + defer f.bucketOKMu.Unlock() + if f.bucketOK { + return nil + } + // List something from the bucket to see if it exists. Doing it like this enables the use of a + // service account that only has the "Storage Object Admin" role. See #2193 for details. + + err = f.pacer.Call(func() (bool, error) { + _, err = f.svc.Objects.List(f.bucket).MaxResults(1).Do() + return shouldRetry(err) + }) + if err == nil { + // Bucket already exists + f.bucketOK = true + return nil + } else if gErr, ok := err.(*googleapi.Error); ok { + if gErr.Code != http.StatusNotFound { + return errors.Wrap(err, "failed to get bucket") + } + } else { + return errors.Wrap(err, "failed to get bucket") + } + + if f.opt.ProjectNumber == "" { + return errors.New("can't make bucket without project number") + } + + bucket := storage.Bucket{ + Name: f.bucket, + Location: f.opt.Location, + StorageClass: f.opt.StorageClass, + } + err = f.pacer.Call(func() (bool, error) { + _, err = f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket).PredefinedAcl(f.opt.BucketACL).Do() + return shouldRetry(err) + }) + if err == nil { + f.bucketOK = true + } + return err +} + +// Rmdir deletes the bucket if the fs is at the root +// +// Returns an error if it isn't empty: Error 409: The bucket you tried +// to delete was not empty. +func (f *Fs) Rmdir(dir string) (err error) { + f.bucketOKMu.Lock() + defer f.bucketOKMu.Unlock() + if f.root != "" || dir != "" { + return nil + } + err = f.pacer.Call(func() (bool, error) { + err = f.svc.Buckets.Delete(f.bucket).Do() + return shouldRetry(err) + }) + if err == nil { + f.bucketOK = false + } + return err +} + +// Precision returns the precision +func (f *Fs) Precision() time.Duration { + return time.Nanosecond +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + err := f.Mkdir("") + if err != nil { + return nil, err + } + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + + // Temporary Object under construction + dstObj := &Object{ + fs: f, + remote: remote, + } + + srcBucket := srcObj.fs.bucket + srcObject := srcObj.fs.root + srcObj.remote + dstBucket := f.bucket + dstObject := f.root + remote + var newObject *storage.Object + err = f.pacer.Call(func() (bool, error) { + newObject, err = f.svc.Objects.Copy(srcBucket, srcObject, dstBucket, dstObject, nil).Do() + return shouldRetry(err) + }) + if err != nil { + return nil, err + } + // Set the metadata for the new object while we have it + dstObj.setMetaData(newObject) + return dstObj, nil +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the Md5sum of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.MD5 { + return "", hash.ErrUnsupported + } + return o.md5sum, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.bytes +} + +// setMetaData sets the fs data from a storage.Object +func (o *Object) setMetaData(info *storage.Object) { + o.url = info.MediaLink + o.bytes = int64(info.Size) + o.mimeType = info.ContentType + + // Read md5sum + md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash) + if err != nil { + fs.Logf(o, "Bad MD5 decode: %v", err) + } else { + o.md5sum = hex.EncodeToString(md5sumData) + } + + // read mtime out of metadata if available + mtimeString, ok := info.Metadata[metaMtime] + if ok { + modTime, err := time.Parse(timeFormatIn, mtimeString) + if err == nil { + o.modTime = modTime + return + } + fs.Debugf(o, "Failed to read mtime from metadata: %s", err) + } + + // Fallback to the Updated time + modTime, err := time.Parse(timeFormatIn, info.Updated) + if err != nil { + fs.Logf(o, "Bad time decode: %v", err) + } else { + o.modTime = modTime + } +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData() (err error) { + if !o.modTime.IsZero() { + return nil + } + var object *storage.Object + err = o.fs.pacer.Call(func() (bool, error) { + object, err = o.fs.svc.Objects.Get(o.fs.bucket, o.fs.root+o.remote).Do() + return shouldRetry(err) + }) + if err != nil { + if gErr, ok := err.(*googleapi.Error); ok { + if gErr.Code == http.StatusNotFound { + return fs.ErrorObjectNotFound + } + } + return err + } + o.setMetaData(object) + return nil +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + // fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// Returns metadata for an object +func metadataFromModTime(modTime time.Time) map[string]string { + metadata := make(map[string]string, 1) + metadata[metaMtime] = modTime.Format(timeFormatOut) + return metadata +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) (err error) { + // This only adds metadata so will perserve other metadata + object := storage.Object{ + Bucket: o.fs.bucket, + Name: o.fs.root + o.remote, + Metadata: metadataFromModTime(modTime), + } + var newObject *storage.Object + err = o.fs.pacer.Call(func() (bool, error) { + newObject, err = o.fs.svc.Objects.Patch(o.fs.bucket, o.fs.root+o.remote, &object).Do() + return shouldRetry(err) + }) + if err != nil { + return err + } + o.setMetaData(newObject) + return nil +} + +// Storable returns a boolean as to whether this object is storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + req, err := http.NewRequest("GET", o.url, nil) + if err != nil { + return nil, err + } + fs.OpenOptionAddHTTPHeaders(req.Header, options) + var res *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + res, err = o.fs.client.Do(req) + if err == nil { + err = googleapi.CheckResponse(res) + if err != nil { + _ = res.Body.Close() // ignore error + } + } + return shouldRetry(err) + }) + if err != nil { + return nil, err + } + _, isRanging := req.Header["Range"] + if !(res.StatusCode == http.StatusOK || (isRanging && res.StatusCode == http.StatusPartialContent)) { + _ = res.Body.Close() // ignore error + return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status) + } + return res.Body, nil +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + err := o.fs.Mkdir("") + if err != nil { + return err + } + modTime := src.ModTime() + + object := storage.Object{ + Bucket: o.fs.bucket, + Name: o.fs.root + o.remote, + ContentType: fs.MimeType(src), + Updated: modTime.Format(timeFormatOut), // Doesn't get set + Metadata: metadataFromModTime(modTime), + } + var newObject *storage.Object + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + newObject, err = o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.opt.ObjectACL).Do() + return shouldRetry(err) + }) + if err != nil { + return err + } + // Set the metadata for the new object while we have it + o.setMetaData(newObject) + return nil +} + +// Remove an object +func (o *Object) Remove() (err error) { + err = o.fs.pacer.Call(func() (bool, error) { + err = o.fs.svc.Objects.Delete(o.fs.bucket, o.fs.root+o.remote).Do() + return shouldRetry(err) + }) + return err +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + return o.mimeType +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.Copier = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.ListRer = &Fs{} + _ fs.Object = &Object{} + _ fs.MimeTyper = &Object{} +) diff --git a/.rclone_repo/backend/googlecloudstorage/googlecloudstorage_test.go b/.rclone_repo/backend/googlecloudstorage/googlecloudstorage_test.go new file mode 100755 index 0000000..5541e11 --- /dev/null +++ b/.rclone_repo/backend/googlecloudstorage/googlecloudstorage_test.go @@ -0,0 +1,17 @@ +// Test GoogleCloudStorage filesystem interface +package googlecloudstorage_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/googlecloudstorage" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestGoogleCloudStorage:", + NilObject: (*googlecloudstorage.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/http/http.go b/.rclone_repo/backend/http/http.go new file mode 100755 index 0000000..cb28f25 --- /dev/null +++ b/.rclone_repo/backend/http/http.go @@ -0,0 +1,503 @@ +// Package http provides a filesystem interface using golang.org/net/http +// +// It treats HTML pages served from the endpoint as directory +// listings, and includes any links found as files. +package http + +import ( + "io" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" + "golang.org/x/net/html" +) + +var ( + errorReadOnly = errors.New("http remotes are read only") + timeUnset = time.Unix(0, 0) +) + +func init() { + fsi := &fs.RegInfo{ + Name: "http", + Description: "http Connection", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "url", + Help: "URL of http host to connect to", + Required: true, + Examples: []fs.OptionExample{{ + Value: "https://example.com", + Help: "Connect to example.com", + }}, + }}, + } + fs.Register(fsi) +} + +// Options defines the configuration for this backend +type Options struct { + Endpoint string `config:"url"` +} + +// Fs stores the interface to the remote HTTP files +type Fs struct { + name string + root string + features *fs.Features // optional features + opt Options // options for this backend + endpoint *url.URL + endpointURL string // endpoint as a string + httpClient *http.Client +} + +// Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading) +type Object struct { + fs *Fs + remote string + size int64 + modTime time.Time + contentType string +} + +// statusError returns an error if the res contained an error +func statusError(res *http.Response, err error) error { + if err != nil { + return err + } + if res.StatusCode < 200 || res.StatusCode > 299 { + _ = res.Body.Close() + return errors.Errorf("HTTP Error %d: %s", res.StatusCode, res.Status) + } + return nil +} + +// NewFs creates a new Fs object from the name and root. It connects to +// the host specified in the config file. +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + if !strings.HasSuffix(opt.Endpoint, "/") { + opt.Endpoint += "/" + } + + // Parse the endpoint and stick the root onto it + base, err := url.Parse(opt.Endpoint) + if err != nil { + return nil, err + } + u, err := rest.URLJoin(base, rest.URLPathEscape(root)) + if err != nil { + return nil, err + } + + client := fshttp.NewClient(fs.Config) + + var isFile = false + if !strings.HasSuffix(u.String(), "/") { + // Make a client which doesn't follow redirects so the server + // doesn't redirect http://host/dir to http://host/dir/ + noRedir := *client + noRedir.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + // check to see if points to a file + res, err := noRedir.Head(u.String()) + err = statusError(res, err) + if err == nil { + isFile = true + } + } + + newRoot := u.String() + if isFile { + // Point to the parent if this is a file + newRoot, _ = path.Split(u.String()) + } else { + if !strings.HasSuffix(newRoot, "/") { + newRoot += "/" + } + } + + u, err = url.Parse(newRoot) + if err != nil { + return nil, err + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + httpClient: client, + endpoint: u, + endpointURL: u.String(), + } + f.features = (&fs.Features{ + CanHaveEmptyDirectories: true, + }).Fill(f) + if isFile { + return f, fs.ErrorIsFile + } + if !strings.HasSuffix(f.endpointURL, "/") { + return nil, errors.New("internal error: url doesn't end with /") + } + return f, nil +} + +// Name returns the configured name of the file system +func (f *Fs) Name() string { + return f.name +} + +// Root returns the root for the filesystem +func (f *Fs) Root() string { + return f.root +} + +// String returns the URL for the filesystem +func (f *Fs) String() string { + return f.endpointURL +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Precision is the remote http file system's modtime precision, which we have no way of knowing. We estimate at 1s +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// NewObject creates a new remote http file object +func (f *Fs) NewObject(remote string) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + err := o.stat() + if err != nil { + return nil, errors.Wrap(err, "Stat failed") + } + return o, nil +} + +// Join's the remote onto the base URL +func (f *Fs) url(remote string) string { + return f.endpointURL + rest.URLPathEscape(remote) +} + +// parse s into an int64, on failure return def +func parseInt64(s string, def int64) int64 { + n, e := strconv.ParseInt(s, 10, 64) + if e != nil { + return def + } + return n +} + +// Errors returned by parseName +var ( + errURLJoinFailed = errors.New("URLJoin failed") + errFoundQuestionMark = errors.New("found ? in URL") + errHostMismatch = errors.New("host mismatch") + errSchemeMismatch = errors.New("scheme mismatch") + errNotUnderRoot = errors.New("not under root") + errNameIsEmpty = errors.New("name is empty") + errNameContainsSlash = errors.New("name contains /") +) + +// parseName turns a name as found in the page into a remote path or returns an error +func parseName(base *url.URL, name string) (string, error) { + // make URL absolute + u, err := rest.URLJoin(base, name) + if err != nil { + return "", errURLJoinFailed + } + // check it doesn't have URL parameters + uStr := u.String() + if strings.Index(uStr, "?") >= 0 { + return "", errFoundQuestionMark + } + // check that this is going back to the same host and scheme + if base.Host != u.Host { + return "", errHostMismatch + } + if base.Scheme != u.Scheme { + return "", errSchemeMismatch + } + // check has path prefix + if !strings.HasPrefix(u.Path, base.Path) { + return "", errNotUnderRoot + } + // calculate the name relative to the base + name = u.Path[len(base.Path):] + // musn't be empty + if name == "" { + return "", errNameIsEmpty + } + // mustn't contain a / - we are looking for a single level directory + slash := strings.Index(name, "/") + if slash >= 0 && slash != len(name)-1 { + return "", errNameContainsSlash + } + return name, nil +} + +// Parse turns HTML for a directory into names +// base should be the base URL to resolve any relative names from +func parse(base *url.URL, in io.Reader) (names []string, err error) { + doc, err := html.Parse(in) + if err != nil { + return nil, err + } + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + for _, a := range n.Attr { + if a.Key == "href" { + name, err := parseName(base, a.Val) + if err == nil { + names = append(names, name) + } + break + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + return names, nil +} + +// Read the directory passed in +func (f *Fs) readDir(dir string) (names []string, err error) { + URL := f.url(dir) + u, err := url.Parse(URL) + if err != nil { + return nil, errors.Wrap(err, "failed to readDir") + } + if !strings.HasSuffix(URL, "/") { + return nil, errors.Errorf("internal error: readDir URL %q didn't end in /", URL) + } + res, err := f.httpClient.Get(URL) + if err == nil && res.StatusCode == http.StatusNotFound { + return nil, fs.ErrorDirNotFound + } + err = statusError(res, err) + if err != nil { + return nil, errors.Wrap(err, "failed to readDir") + } + defer fs.CheckClose(res.Body, &err) + + contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0] + switch contentType { + case "text/html": + names, err = parse(u, res.Body) + if err != nil { + return nil, errors.Wrap(err, "readDir") + } + default: + return nil, errors.Errorf("Can't parse content type %q", contentType) + } + return names, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + if !strings.HasSuffix(dir, "/") && dir != "" { + dir += "/" + } + names, err := f.readDir(dir) + if err != nil { + return nil, errors.Wrapf(err, "error listing %q", dir) + } + for _, name := range names { + isDir := name[len(name)-1] == '/' + name = strings.TrimRight(name, "/") + remote := path.Join(dir, name) + if isDir { + dir := fs.NewDir(remote, timeUnset) + entries = append(entries, dir) + } else { + file := &Object{ + fs: f, + remote: remote, + } + if err = file.stat(); err != nil { + fs.Debugf(remote, "skipping because of error: %v", err) + continue + } + entries = append(entries, file) + } + } + return entries, nil +} + +// Put in to the remote path with the modTime given of the given size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return nil, errorReadOnly +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return nil, errorReadOnly +} + +// Fs is the filesystem this remote http file object is located within +func (o *Object) Fs() fs.Info { + return o.fs +} + +// String returns the URL to the remote HTTP file +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote the name of the remote HTTP file, relative to the fs root +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns "" since HTTP (in Go or OpenSSH) doesn't support remote calculation of hashes +func (o *Object) Hash(r hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// Size returns the size in bytes of the remote http file +func (o *Object) Size() int64 { + return o.size +} + +// ModTime returns the modification time of the remote http file +func (o *Object) ModTime() time.Time { + return o.modTime +} + +// url returns the native url of the object +func (o *Object) url() string { + return o.fs.url(o.remote) +} + +// stat updates the info field in the Object +func (o *Object) stat() error { + url := o.url() + res, err := o.fs.httpClient.Head(url) + err = statusError(res, err) + if err != nil { + return errors.Wrap(err, "failed to stat") + } + t, err := http.ParseTime(res.Header.Get("Last-Modified")) + if err != nil { + t = timeUnset + } + o.size = parseInt64(res.Header.Get("Content-Length"), -1) + o.modTime = t + o.contentType = res.Header.Get("Content-Type") + return nil +} + +// SetModTime sets the modification and access time to the specified time +// +// it also updates the info field +func (o *Object) SetModTime(modTime time.Time) error { + return errorReadOnly +} + +// Storable returns whether the remote http file is a regular file (not a directory, symbolic link, block device, character device, named pipe, etc) +func (o *Object) Storable() bool { + return true +} + +// Open a remote http file object for reading. Seek is supported +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + url := o.url() + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, errors.Wrap(err, "Open failed") + } + + // Add optional headers + for k, v := range fs.OpenOptionHeaders(options) { + req.Header.Add(k, v) + } + + // Do the request + res, err := o.fs.httpClient.Do(req) + err = statusError(res, err) + if err != nil { + return nil, errors.Wrap(err, "Open failed") + } + return res.Body, nil +} + +// Hashes returns hash.HashNone to indicate remote hashing is unavailable +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.None) +} + +// Mkdir makes the root directory of the Fs object +func (f *Fs) Mkdir(dir string) error { + return errorReadOnly +} + +// Remove a remote http file object +func (o *Object) Remove() error { + return errorReadOnly +} + +// Rmdir removes the root directory of the Fs object +func (f *Fs) Rmdir(dir string) error { + return errorReadOnly +} + +// Update in to the object with the modTime given of the given size +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + return errorReadOnly +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + return o.contentType +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.Object = &Object{} + _ fs.MimeTyper = &Object{} +) diff --git a/.rclone_repo/backend/http/http_internal_test.go b/.rclone_repo/backend/http/http_internal_test.go new file mode 100755 index 0000000..bb062ca --- /dev/null +++ b/.rclone_repo/backend/http/http_internal_test.go @@ -0,0 +1,327 @@ +// +build go1.8 + +package http + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fstest" + "github.com/ncw/rclone/lib/rest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + remoteName = "TestHTTP" + testPath = "test" + filesPath = filepath.Join(testPath, "files") +) + +// prepareServer the test server and return a function to tidy it up afterwards +func prepareServer(t *testing.T) (configmap.Simple, func()) { + // file server for test/files + fileServer := http.FileServer(http.Dir(filesPath)) + + // Make the test server + ts := httptest.NewServer(fileServer) + + // Configure the remote + config.LoadConfig() + // fs.Config.LogLevel = fs.LogLevelDebug + // fs.Config.DumpHeaders = true + // fs.Config.DumpBodies = true + // config.FileSet(remoteName, "type", "http") + // config.FileSet(remoteName, "url", ts.URL) + + m := configmap.Simple{ + "type": "http", + "url": ts.URL, + } + + // return a function to tidy up + return m, ts.Close +} + +// prepare the test server and return a function to tidy it up afterwards +func prepare(t *testing.T) (fs.Fs, func()) { + m, tidy := prepareServer(t) + + // Instantiate it + f, err := NewFs(remoteName, "", m) + require.NoError(t, err) + + return f, tidy +} + +func testListRoot(t *testing.T, f fs.Fs) { + entries, err := f.List("") + require.NoError(t, err) + + sort.Sort(entries) + + require.Equal(t, 4, len(entries)) + + e := entries[0] + assert.Equal(t, "four", e.Remote()) + assert.Equal(t, int64(-1), e.Size()) + _, ok := e.(fs.Directory) + assert.True(t, ok) + + e = entries[1] + assert.Equal(t, "one%.txt", e.Remote()) + assert.Equal(t, int64(6), e.Size()) + _, ok = e.(*Object) + assert.True(t, ok) + + e = entries[2] + assert.Equal(t, "three", e.Remote()) + assert.Equal(t, int64(-1), e.Size()) + _, ok = e.(fs.Directory) + assert.True(t, ok) + + e = entries[3] + assert.Equal(t, "two.html", e.Remote()) + assert.Equal(t, int64(7), e.Size()) + _, ok = e.(*Object) + assert.True(t, ok) +} + +func TestListRoot(t *testing.T) { + f, tidy := prepare(t) + defer tidy() + testListRoot(t, f) +} + +func TestListSubDir(t *testing.T) { + f, tidy := prepare(t) + defer tidy() + + entries, err := f.List("three") + require.NoError(t, err) + + sort.Sort(entries) + + assert.Equal(t, 1, len(entries)) + + e := entries[0] + assert.Equal(t, "three/underthree.txt", e.Remote()) + assert.Equal(t, int64(9), e.Size()) + _, ok := e.(*Object) + assert.True(t, ok) +} + +func TestNewObject(t *testing.T) { + f, tidy := prepare(t) + defer tidy() + + o, err := f.NewObject("four/under four.txt") + require.NoError(t, err) + + assert.Equal(t, "four/under four.txt", o.Remote()) + assert.Equal(t, int64(9), o.Size()) + _, ok := o.(*Object) + assert.True(t, ok) + + // Test the time is correct on the object + + tObj := o.ModTime() + + fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt")) + require.NoError(t, err) + tFile := fi.ModTime() + + dt, ok := fstest.CheckTimeEqualWithPrecision(tObj, tFile, time.Second) + assert.True(t, ok, fmt.Sprintf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", o.Remote(), dt, time.Second, tObj, tFile, time.Second)) +} + +func TestOpen(t *testing.T) { + f, tidy := prepare(t) + defer tidy() + + o, err := f.NewObject("four/under four.txt") + require.NoError(t, err) + + // Test normal read + fd, err := o.Open() + require.NoError(t, err) + data, err := ioutil.ReadAll(fd) + require.NoError(t, err) + require.NoError(t, fd.Close()) + assert.Equal(t, "beetroot\n", string(data)) + + // Test with range request + fd, err = o.Open(&fs.RangeOption{Start: 1, End: 5}) + require.NoError(t, err) + data, err = ioutil.ReadAll(fd) + require.NoError(t, err) + require.NoError(t, fd.Close()) + assert.Equal(t, "eetro", string(data)) +} + +func TestMimeType(t *testing.T) { + f, tidy := prepare(t) + defer tidy() + + o, err := f.NewObject("four/under four.txt") + require.NoError(t, err) + + do, ok := o.(fs.MimeTyper) + require.True(t, ok) + assert.Equal(t, "text/plain; charset=utf-8", do.MimeType()) +} + +func TestIsAFileRoot(t *testing.T) { + m, tidy := prepareServer(t) + defer tidy() + + f, err := NewFs(remoteName, "one%.txt", m) + assert.Equal(t, err, fs.ErrorIsFile) + + testListRoot(t, f) +} + +func TestIsAFileSubDir(t *testing.T) { + m, tidy := prepareServer(t) + defer tidy() + + f, err := NewFs(remoteName, "three/underthree.txt", m) + assert.Equal(t, err, fs.ErrorIsFile) + + entries, err := f.List("") + require.NoError(t, err) + + sort.Sort(entries) + + assert.Equal(t, 1, len(entries)) + + e := entries[0] + assert.Equal(t, "underthree.txt", e.Remote()) + assert.Equal(t, int64(9), e.Size()) + _, ok := e.(*Object) + assert.True(t, ok) +} + +func TestParseName(t *testing.T) { + for i, test := range []struct { + base string + val string + wantErr error + want string + }{ + {"http://example.com/", "potato", nil, "potato"}, + {"http://example.com/dir/", "potato", nil, "potato"}, + {"http://example.com/dir/", "potato?download=true", errFoundQuestionMark, ""}, + {"http://example.com/dir/", "../dir/potato", nil, "potato"}, + {"http://example.com/dir/", "..", errNotUnderRoot, ""}, + {"http://example.com/dir/", "http://example.com/", errNotUnderRoot, ""}, + {"http://example.com/dir/", "http://example.com/dir/", errNameIsEmpty, ""}, + {"http://example.com/dir/", "http://example.com/dir/potato", nil, "potato"}, + {"http://example.com/dir/", "https://example.com/dir/potato", errSchemeMismatch, ""}, + {"http://example.com/dir/", "http://notexample.com/dir/potato", errHostMismatch, ""}, + {"http://example.com/dir/", "/dir/", errNameIsEmpty, ""}, + {"http://example.com/dir/", "/dir/potato", nil, "potato"}, + {"http://example.com/dir/", "subdir/potato", errNameContainsSlash, ""}, + {"http://example.com/dir/", "With percent %25.txt", nil, "With percent %.txt"}, + {"http://example.com/dir/", "With colon :", errURLJoinFailed, ""}, + {"http://example.com/dir/", rest.URLPathEscape("With colon :"), nil, "With colon :"}, + {"http://example.com/Dungeons%20%26%20Dragons/", "/Dungeons%20&%20Dragons/D%26D%20Basic%20%28Holmes%2C%20B%2C%20X%2C%20BECMI%29/", nil, "D&D Basic (Holmes, B, X, BECMI)/"}, + } { + u, err := url.Parse(test.base) + require.NoError(t, err) + got, gotErr := parseName(u, test.val) + what := fmt.Sprintf("test %d base=%q, val=%q", i, test.base, test.val) + assert.Equal(t, test.wantErr, gotErr, what) + assert.Equal(t, test.want, got, what) + } +} + +// Load HTML from the file given and parse it, checking it against the entries passed in +func parseHTML(t *testing.T, name string, base string, want []string) { + in, err := os.Open(filepath.Join(testPath, "index_files", name)) + require.NoError(t, err) + defer func() { + require.NoError(t, in.Close()) + }() + if base == "" { + base = "http://example.com/" + } + u, err := url.Parse(base) + require.NoError(t, err) + entries, err := parse(u, in) + require.NoError(t, err) + assert.Equal(t, want, entries) +} + +func TestParseEmpty(t *testing.T) { + parseHTML(t, "empty.html", "", []string(nil)) +} + +func TestParseApache(t *testing.T) { + parseHTML(t, "apache.html", "http://example.com/nick/pub/", []string{ + "SWIG-embed.tar.gz", + "avi2dvd.pl", + "cambert.exe", + "cambert.gz", + "fedora_demo.gz", + "gchq-challenge/", + "mandelterm/", + "pgp-key.txt", + "pymath/", + "rclone", + "readdir.exe", + "rush_hour_solver_cut_down.py", + "snake-puzzle/", + "stressdisk/", + "timer-test", + "words-to-regexp.pl", + "Now 100% better.mp3", + "Now better.mp3", + }) +} + +func TestParseMemstore(t *testing.T) { + parseHTML(t, "memstore.html", "", []string{ + "test/", + "v1.35/", + "v1.36-01-g503cd84/", + "rclone-beta-latest-freebsd-386.zip", + "rclone-beta-latest-freebsd-amd64.zip", + "rclone-beta-latest-windows-amd64.zip", + }) +} + +func TestParseNginx(t *testing.T) { + parseHTML(t, "nginx.html", "", []string{ + "deltas/", + "objects/", + "refs/", + "state/", + "config", + "summary", + }) +} + +func TestParseCaddy(t *testing.T) { + parseHTML(t, "caddy.html", "", []string{ + "mimetype.zip", + "rclone-delete-empty-dirs.py", + "rclone-show-empty-dirs.py", + "stat-windows-386.zip", + "v1.36-155-gcf29ee8b-team-driveβ/", + "v1.36-156-gca76b3fb-team-driveβ/", + "v1.36-156-ge1f0e0f5-team-driveβ/", + "v1.36-22-g06ea13a-ssh-agentβ/", + }) +} diff --git a/.rclone_repo/backend/http/test/files/four/under four.txt b/.rclone_repo/backend/http/test/files/four/under four.txt new file mode 100755 index 0000000..748393c --- /dev/null +++ b/.rclone_repo/backend/http/test/files/four/under four.txt @@ -0,0 +1 @@ +beetroot diff --git a/.rclone_repo/backend/http/test/files/one%.txt b/.rclone_repo/backend/http/test/files/one%.txt new file mode 100755 index 0000000..ce01362 --- /dev/null +++ b/.rclone_repo/backend/http/test/files/one%.txt @@ -0,0 +1 @@ +hello diff --git a/.rclone_repo/backend/http/test/files/three/underthree.txt b/.rclone_repo/backend/http/test/files/three/underthree.txt new file mode 100755 index 0000000..1031dc5 --- /dev/null +++ b/.rclone_repo/backend/http/test/files/three/underthree.txt @@ -0,0 +1 @@ +rutabaga diff --git a/.rclone_repo/backend/http/test/files/two.html b/.rclone_repo/backend/http/test/files/two.html new file mode 100755 index 0000000..4bc5628 --- /dev/null +++ b/.rclone_repo/backend/http/test/files/two.html @@ -0,0 +1 @@ +potato diff --git a/.rclone_repo/backend/http/test/index_files/apache.html b/.rclone_repo/backend/http/test/index_files/apache.html new file mode 100755 index 0000000..ba754ad --- /dev/null +++ b/.rclone_repo/backend/http/test/index_files/apache.html @@ -0,0 +1,32 @@ + + + + Index of /nick/pub + + +

Index of /nick/pub

+ + + + + + + + + + + + + + + + + + + + + + + +
[ICO]NameLast modifiedSizeDescription
[DIR]Parent Directory  -  
[ ]SWIG-embed.tar.gz29-Nov-2005 16:27 2.3K 
[TXT]avi2dvd.pl14-Apr-2010 23:07 17K 
[ ]cambert.exe15-Dec-2006 18:07 54K 
[ ]cambert.gz14-Apr-2010 23:07 18K 
[ ]fedora_demo.gz08-Jun-2007 11:01 1.0M 
[DIR]gchq-challenge/24-Dec-2016 15:24 -  
[DIR]mandelterm/13-Jul-2013 22:22 -  
[TXT]pgp-key.txt14-Apr-2010 23:07 400  
[DIR]pymath/24-Dec-2016 15:24 -  
[ ]rclone09-May-2017 17:15 22M 
[ ]readdir.exe21-Oct-2016 14:47 1.6M 
[TXT]rush_hour_solver_cut_down.py23-Jul-2009 11:44 14K 
[DIR]snake-puzzle/25-Sep-2016 20:56 -  
[DIR]stressdisk/08-Nov-2016 14:25 -  
[ ]timer-test09-May-2017 17:05 1.5M 
[TXT]words-to-regexp.pl01-Mar-2005 20:43 6.0K 
[SND]Now 100% better.mp32017-08-01 11:41 0  
[SND]Now better.mp32017-08-01 11:41 0  
+ diff --git a/.rclone_repo/backend/http/test/index_files/caddy.html b/.rclone_repo/backend/http/test/index_files/caddy.html new file mode 100755 index 0000000..bd7250a --- /dev/null +++ b/.rclone_repo/backend/http/test/index_files/caddy.html @@ -0,0 +1,378 @@ + + + + / + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

+ / +

+
+
+
+
+ 4 directories + 4 files + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Name + + Size + + Modified +
+ + + mimetype.zip + + 765 KiB
+ + + rclone-delete-empty-dirs.py + + 1.2 KiB
+ + + rclone-show-empty-dirs.py + + 868 B
+ + + stat-windows-386.zip + + 688 KiB
+ + + v1.36-155-gcf29ee8b-team-driveβ + + —
+ + + v1.36-156-gca76b3fb-team-driveβ + + —
+ + + v1.36-156-ge1f0e0f5-team-driveβ + + —
+ + + v1.36-22-g06ea13a-ssh-agentβ + + —
+
+
+ + + + + diff --git a/.rclone_repo/backend/http/test/index_files/empty.html b/.rclone_repo/backend/http/test/index_files/empty.html new file mode 100755 index 0000000..e69de29 diff --git a/.rclone_repo/backend/http/test/index_files/memstore.html b/.rclone_repo/backend/http/test/index_files/memstore.html new file mode 100755 index 0000000..7616cad --- /dev/null +++ b/.rclone_repo/backend/http/test/index_files/memstore.html @@ -0,0 +1,77 @@ + + + + + + Index of / + + +
+

Index of /

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeSizeLast modifiedMD5
test/application/directory0 bytes--
v1.35/application/directory0 bytes--
v1.36-01-g503cd84/application/directory0 bytes--
rclone-beta-latest-freebsd-386.zipapplication/zip4.6 MB2017-06-19 14:04:52e747003c69c81e675f206a715264bfa8
rclone-beta-latest-freebsd-amd64.zipapplication/zip5.0 MB2017-06-19 14:04:53ff30b5e9bf2863a2373069142e6f2b7f
rclone-beta-latest-windows-amd64.zipapplication/x-zip-compressed4.9 MB2017-06-19 13:56:02851a5547a0495cbbd94cbc90a80ed6f5
+

Memset Ltd.

+
+ + diff --git a/.rclone_repo/backend/http/test/index_files/nginx.html b/.rclone_repo/backend/http/test/index_files/nginx.html new file mode 100755 index 0000000..850f7f0 --- /dev/null +++ b/.rclone_repo/backend/http/test/index_files/nginx.html @@ -0,0 +1,12 @@ + +Index of /atomic/fedora/ + +

Index of /atomic/fedora/

../
+deltas/                                            04-May-2017 21:37                   -
+objects/                                           04-May-2017 20:44                   -
+refs/                                              04-May-2017 20:42                   -
+state/                                             04-May-2017 21:36                   -
+config                                             04-May-2017 20:42                 118
+summary                                            04-May-2017 21:36                 806
+
+ diff --git a/.rclone_repo/backend/hubic/auth.go b/.rclone_repo/backend/hubic/auth.go new file mode 100755 index 0000000..08469ac --- /dev/null +++ b/.rclone_repo/backend/hubic/auth.go @@ -0,0 +1,61 @@ +package hubic + +import ( + "net/http" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/swift" +) + +// auth is an authenticator for swift +type auth struct { + f *Fs +} + +// newAuth creates a swift authenticator +func newAuth(f *Fs) *auth { + return &auth{ + f: f, + } +} + +// Request constructs a http.Request for authentication +// +// returns nil for not needed +func (a *auth) Request(*swift.Connection) (r *http.Request, err error) { + const retries = 10 + for try := 1; try <= retries; try++ { + err = a.f.getCredentials() + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + fs.Debugf(a.f, "retrying auth request %d/%d: %v", try, retries, err) + } + return nil, err +} + +// Response parses the result of an http request +func (a *auth) Response(resp *http.Response) error { + return nil +} + +// The public storage URL - set Internal to true to read +// internal/service net URL +func (a *auth) StorageUrl(Internal bool) string { // nolint + return a.f.credentials.Endpoint +} + +// The access token +func (a *auth) Token() string { + return a.f.credentials.Token +} + +// The CDN url if available +func (a *auth) CdnUrl() string { // nolint + return "" +} + +// Check the interfaces are satisfied +var _ swift.Authenticator = (*auth)(nil) diff --git a/.rclone_repo/backend/hubic/hubic.go b/.rclone_repo/backend/hubic/hubic.go new file mode 100755 index 0000000..955ab7d --- /dev/null +++ b/.rclone_repo/backend/hubic/hubic.go @@ -0,0 +1,203 @@ +// Package hubic provides an interface to the Hubic object storage +// system. +package hubic + +// This uses the normal swift mechanism to update the credentials and +// ignores the expires field returned by the Hubic API. This may need +// to be revisted after some actual experience. + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "time" + + "github.com/ncw/rclone/backend/swift" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/lib/oauthutil" + swiftLib "github.com/ncw/swift" + "github.com/pkg/errors" + "golang.org/x/oauth2" +) + +const ( + rcloneClientID = "api_hubic_svWP970PvSWbw5G3PzrAqZ6X2uHeZBPI" + rcloneEncryptedClientSecret = "leZKCcqy9movLhDWLVXX8cSLp_FzoiAPeEJOIOMRw1A5RuC4iLEPDYPWVF46adC_MVonnLdVEOTHVstfBOZ_lY4WNp8CK_YWlpRZ9diT5YI" +) + +// Globals +var ( + // Description of how to auth for this app + oauthConfig = &oauth2.Config{ + Scopes: []string{ + "credentials.r", // Read Openstack credentials + }, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://api.hubic.com/oauth/auth/", + TokenURL: "https://api.hubic.com/oauth/token/", + }, + ClientID: rcloneClientID, + ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), + RedirectURL: oauthutil.RedirectLocalhostURL, + } +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "hubic", + Description: "Hubic", + NewFs: NewFs, + Config: func(name string, m configmap.Mapper) { + err := oauthutil.Config("hubic", name, m, oauthConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + }, + Options: append([]fs.Option{{ + Name: config.ConfigClientID, + Help: "Hubic Client Id\nLeave blank normally.", + }, { + Name: config.ConfigClientSecret, + Help: "Hubic Client Secret\nLeave blank normally.", + }}, swift.SharedOptions...), + }) +} + +// credentials is the JSON returned from the Hubic API to read the +// OpenStack credentials +type credentials struct { + Token string `json:"token"` // Openstack token + Endpoint string `json:"endpoint"` // Openstack endpoint + Expires string `json:"expires"` // Expires date - eg "2015-11-09T14:24:56+01:00" +} + +// Fs represents a remote hubic +type Fs struct { + fs.Fs // wrapped Fs + features *fs.Features // optional features + client *http.Client // client for oauth api + credentials credentials // returned from the Hubic API + expires time.Time // time credentials expire +} + +// Object describes a swift object +type Object struct { + *swift.Object +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.Object.String() +} + +// ------------------------------------------------------------ + +// String converts this Fs to a string +func (f *Fs) String() string { + if f.Fs == nil { + return "Hubic" + } + return fmt.Sprintf("Hubic %s", f.Fs.String()) +} + +// getCredentials reads the OpenStack Credentials using the Hubic API +// +// The credentials are read into the Fs +func (f *Fs) getCredentials() (err error) { + req, err := http.NewRequest("GET", "https://api.hubic.com/1.0/account/credentials", nil) + if err != nil { + return err + } + resp, err := f.client.Do(req) + if err != nil { + return err + } + defer fs.CheckClose(resp.Body, &err) + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return errors.Errorf("failed to get credentials: %s", resp.Status) + } + decoder := json.NewDecoder(resp.Body) + var result credentials + err = decoder.Decode(&result) + if err != nil { + return err + } + // fs.Debugf(f, "Got credentials %+v", result) + if result.Token == "" || result.Endpoint == "" || result.Expires == "" { + return errors.New("couldn't read token, result and expired from credentials") + } + f.credentials = result + expires, err := time.Parse(time.RFC3339, result.Expires) + if err != nil { + return err + } + f.expires = expires + fs.Debugf(f, "Got swift credentials (expiry %v in %v)", f.expires, f.expires.Sub(time.Now())) + return nil +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + client, _, err := oauthutil.NewClient(name, m, oauthConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to configure Hubic") + } + + f := &Fs{ + client: client, + } + + // Make the swift Connection + c := &swiftLib.Connection{ + Auth: newAuth(f), + ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport + Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport + Transport: fshttp.NewTransport(fs.Config), + } + err = c.Authenticate() + if err != nil { + return nil, errors.Wrap(err, "error authenticating swift connection") + } + + // Parse config into swift.Options struct + opt := new(swift.Options) + err = configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + // Make inner swift Fs from the connection + swiftFs, err := swift.NewFsWithConnection(opt, name, root, c, true) + if err != nil && err != fs.ErrorIsFile { + return nil, err + } + f.Fs = swiftFs + f.features = f.Fs.Features().Wrap(f) + return f, err +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// UnWrap returns the Fs that this Fs is wrapping +func (f *Fs) UnWrap() fs.Fs { + return f.Fs +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.UnWrapper = (*Fs)(nil) +) diff --git a/.rclone_repo/backend/hubic/hubic_test.go b/.rclone_repo/backend/hubic/hubic_test.go new file mode 100755 index 0000000..8176f0a --- /dev/null +++ b/.rclone_repo/backend/hubic/hubic_test.go @@ -0,0 +1,17 @@ +// Test Hubic filesystem interface +package hubic_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/hubic" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestHubic:", + NilObject: (*hubic.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/jottacloud/api/types.go b/.rclone_repo/backend/jottacloud/api/types.go new file mode 100755 index 0000000..22d7d71 --- /dev/null +++ b/.rclone_repo/backend/jottacloud/api/types.go @@ -0,0 +1,266 @@ +package api + +import ( + "encoding/xml" + "fmt" + "time" + + "github.com/pkg/errors" +) + +const ( + timeFormat = "2006-01-02-T15:04:05Z0700" +) + +// Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format. +type Time time.Time + +// UnmarshalXML turns XML into a Time +func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var v string + if err := d.DecodeElement(&v, &start); err != nil { + return err + } + if v == "" { + *t = Time(time.Time{}) + return nil + } + newTime, err := time.Parse(timeFormat, v) + if err == nil { + *t = Time(newTime) + } + return err +} + +// MarshalXML turns a Time into XML +func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + return e.EncodeElement(t.String(), start) +} + +// Return Time string in Jottacloud format +func (t Time) String() string { return time.Time(t).Format(timeFormat) } + +// Flag is a hacky type for checking if an attribute is present +type Flag bool + +// UnmarshalXMLAttr sets Flag to true if the attribute is present +func (f *Flag) UnmarshalXMLAttr(attr xml.Attr) error { + *f = true + return nil +} + +// MarshalXMLAttr : Do not use +func (f *Flag) MarshalXMLAttr(name xml.Name) (xml.Attr, error) { + attr := xml.Attr{ + Name: name, + Value: "false", + } + return attr, errors.New("unimplemented") +} + +/* +GET http://www.jottacloud.com/JFS/ + + + 12qh1wsht8cssxdtwl15rqh9 + free + false + 5368709120 + -1 + -1 + 0 + false + false + false + true + true + + + Jotta + Jotta + JOTTA + 5c458d01-9eaf-4f23-8d3c-2486fd9704d8 + 0 + 2018-07-15-T22:04:59Z + + + +*/ + +// AccountInfo represents a Jottacloud account +type AccountInfo struct { + Username string `xml:"username"` + AccountType string `xml:"account-type"` + Locked bool `xml:"locked"` + Capacity int64 `xml:"capacity"` + MaxDevices int `xml:"max-devices"` + MaxMobileDevices int `xml:"max-mobile-devices"` + Usage int64 `xml:"usage"` + ReadLocked bool `xml:"read-locked"` + WriteLocked bool `xml:"write-locked"` + QuotaWriteLocked bool `xml:"quota-write-locked"` + EnableSync bool `xml:"enable-sync"` + EnableFolderShare bool `xml:"enable-foldershare"` + Devices []JottaDevice `xml:"devices>device"` +} + +/* +GET http://www.jottacloud.com/JFS// + + + Jotta + Jotta + JOTTA + 5c458d01-9eaf-4f23-8d3c-2486fd9704d8 + 0 + 2018-07-15-T22:04:59Z + 12qh1wsht8cssxdtwl15rqh9 + + + Archive + 0 + 2018-07-15-T22:04:59Z + + + Shared + 0 + + + + Sync + 0 + + + + + +*/ + +// JottaDevice represents a Jottacloud Device +type JottaDevice struct { + Name string `xml:"name"` + DisplayName string `xml:"display_name"` + Type string `xml:"type"` + Sid string `xml:"sid"` + Size int64 `xml:"size"` + User string `xml:"user"` + MountPoints []JottaMountPoint `xml:"mountPoints>mountPoint"` +} + +/* +GET http://www.jottacloud.com/JFS/// + + + Sync + /12qh1wsht8cssxdtwl15rqh9/Jotta + /12qh1wsht8cssxdtwl15rqh9/Jotta + 0 + + Jotta + 12qh1wsht8cssxdtwl15rqh9 + + + + + +*/ + +// JottaMountPoint represents a Jottacloud mountpoint +type JottaMountPoint struct { + Name string `xml:"name"` + Size int64 `xml:"size"` + Device string `xml:"device"` + Folders []JottaFolder `xml:"folders>folder"` + Files []JottaFile `xml:"files>file"` +} + +/* +GET http://www.jottacloud.com/JFS//// + + + /12qh1wsht8cssxdtwl15rqh9/Jotta/Sync + /12qh1wsht8cssxdtwl15rqh9/Jotta/Sync + + c + + + + + 1 + COMPLETED + 2018-07-05-T15:08:02Z + 2018-07-05-T15:08:02Z + application/octet-stream + 30827730 + 1e8a7b728ab678048df00075c9507158 + 2018-07-24-T20:41:10Z + + + + + +*/ + +// JottaFolder represents a JottacloudFolder +type JottaFolder struct { + XMLName xml.Name + Name string `xml:"name,attr"` + Deleted Flag `xml:"deleted,attr"` + Path string `xml:"path"` + CreatedAt Time `xml:"created"` + ModifiedAt Time `xml:"modified"` + Updated Time `xml:"updated"` + Folders []JottaFolder `xml:"folders>folder"` + Files []JottaFile `xml:"files>file"` +} + +/* +GET http://www.jottacloud.com/JFS////.../ + + + + 1 + COMPLETED + 2018-07-05-T15:08:02Z + 2018-07-05-T15:08:02Z + application/octet-stream + 30827730 + 1e8a7b728ab678048df00075c9507158 + 2018-07-24-T20:41:10Z + + +*/ + +// JottaFile represents a Jottacloud file +type JottaFile struct { + XMLName xml.Name + Name string `xml:"name,attr"` + Deleted Flag `xml:"deleted,attr"` + State string `xml:"currentRevision>state"` + CreatedAt Time `xml:"currentRevision>created"` + ModifiedAt Time `xml:"currentRevision>modified"` + Updated Time `xml:"currentRevision>updated"` + Size int64 `xml:"currentRevision>size"` + MimeType string `xml:"currentRevision>mime"` + MD5 string `xml:"currentRevision>md5"` +} + +// Error is a custom Error for wrapping Jottacloud error responses +type Error struct { + StatusCode int `xml:"code"` + Message string `xml:"message"` + Reason string `xml:"reason"` + Cause string `xml:"cause"` +} + +// Error returns a string for the error and statistifes the error interface +func (e *Error) Error() string { + out := fmt.Sprintf("error %d", e.StatusCode) + if e.Message != "" { + out += ": " + e.Message + } + if e.Reason != "" { + out += fmt.Sprintf(" (%+v)", e.Reason) + } + return out +} diff --git a/.rclone_repo/backend/jottacloud/api/types_test.go b/.rclone_repo/backend/jottacloud/api/types_test.go new file mode 100755 index 0000000..8451504 --- /dev/null +++ b/.rclone_repo/backend/jottacloud/api/types_test.go @@ -0,0 +1,29 @@ +package api + +import ( + "encoding/xml" + "testing" + "time" +) + +func TestMountpointEmptyModificationTime(t *testing.T) { + mountpoint := ` + + Sync + /foo/Jotta + /foo/Jotta + 0 + + Jotta + foo + + +` + var jf JottaFolder + if err := xml.Unmarshal([]byte(mountpoint), &jf); err != nil { + t.Fatal(err) + } + if !time.Time(jf.ModifiedAt).IsZero() { + t.Errorf("got non-zero time, want zero") + } +} diff --git a/.rclone_repo/backend/jottacloud/jottacloud.go b/.rclone_repo/backend/jottacloud/jottacloud.go new file mode 100755 index 0000000..f27c5fc --- /dev/null +++ b/.rclone_repo/backend/jottacloud/jottacloud.go @@ -0,0 +1,935 @@ +package jottacloud + +import ( + "bytes" + "crypto/md5" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "path" + "strconv" + "strings" + "time" + + "github.com/ncw/rclone/backend/jottacloud/api" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/accounting" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/pacer" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" +) + +// Globals +const ( + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential + defaultDevice = "Jotta" + defaultMountpoint = "Sync" + rootURL = "https://www.jottacloud.com/jfs/" + apiURL = "https://api.jottacloud.com" + cachePrefix = "rclone-jcmd5-" +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "jottacloud", + Description: "JottaCloud", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "user", + Help: "User Name", + }, { + Name: "pass", + Help: "Password.", + IsPassword: true, + }, { + Name: "mountpoint", + Help: "The mountpoint to use.", + Required: true, + Examples: []fs.OptionExample{{ + Value: "Sync", + Help: "Will be synced by the official client.", + }, { + Value: "Archive", + Help: "Archive", + }}, + }, { + Name: "md5_memory_limit", + Help: "Files bigger than this will be cached on disk to calculate the MD5 if required.", + Default: fs.SizeSuffix(10 * 1024 * 1024), + Advanced: true, + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + User string `config:"user"` + Pass string `config:"pass"` + Mountpoint string `config:"mountpoint"` + MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"` +} + +// Fs represents a remote jottacloud +type Fs struct { + name string + root string + user string + opt Options + features *fs.Features + endpointURL string + srv *rest.Client + pacer *pacer.Pacer +} + +// Object describes a jottacloud object +// +// Will definitely have info but maybe not meta +type Object struct { + fs *Fs + remote string + hasMetaData bool + size int64 + modTime time.Time + md5 string + mimeType string +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("jottacloud root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// parsePath parses an box 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(resp *http.Response, err error) (bool, error) { + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// readMetaDataForPath reads the metadata from the path +func (f *Fs) readMetaDataForPath(path string) (info *api.JottaFile, err error) { + opts := rest.Opts{ + Method: "GET", + Path: f.filePath(path), + } + var result api.JottaFile + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(&opts, nil, &result) + return shouldRetry(resp, err) + }) + + if apiErr, ok := err.(*api.Error); ok { + // does not exist + if apiErr.StatusCode == http.StatusNotFound { + return nil, fs.ErrorObjectNotFound + } + } + + if err != nil { + return nil, errors.Wrap(err, "read metadata failed") + } + if result.XMLName.Local != "file" { + return nil, fs.ErrorNotAFile + } + return &result, nil +} + +// getAccountInfo retrieves account information +func (f *Fs) getAccountInfo() (info *api.AccountInfo, err error) { + opts := rest.Opts{ + Method: "GET", + Path: rest.URLPathEscape(f.user), + } + + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(&opts, nil, &info) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + + return info, nil +} + +// setEndpointUrl reads the account id and generates the API endpoint URL +func (f *Fs) setEndpointURL(mountpoint string) (err error) { + info, err := f.getAccountInfo() + if err != nil { + return errors.Wrap(err, "failed to get endpoint url") + } + f.endpointURL = rest.URLPathEscape(path.Join(info.Username, defaultDevice, mountpoint)) + return nil +} + +// errorHandler parses a non 2xx error response into an error +func errorHandler(resp *http.Response) error { + // Decode error response + errResponse := new(api.Error) + err := rest.DecodeXML(resp, &errResponse) + if err != nil { + fs.Debugf(nil, "Couldn't decode error response: %v", err) + } + if errResponse.Message == "" { + errResponse.Message = resp.Status + } + if errResponse.StatusCode == 0 { + errResponse.StatusCode = resp.StatusCode + } + return errResponse +} + +// filePath returns a escaped file path (f.root, file) +func (f *Fs) filePath(file string) string { + return rest.URLPathEscape(path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, file)))) +} + +// filePath returns a escaped file path (f.root, remote) +func (o *Object) filePath() string { + return o.fs.filePath(o.remote) +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + rootIsDir := strings.HasSuffix(root, "/") + root = parsePath(root) + + user := config.FileGet(name, "user") + pass := config.FileGet(name, "pass") + + if opt.Pass != "" { + var err error + opt.Pass, err = obscure.Reveal(opt.Pass) + if err != nil { + return nil, errors.Wrap(err, "couldn't decrypt password") + } + } + + f := &Fs{ + name: name, + root: root, + user: opt.User, + opt: *opt, + //endpointURL: rest.URLPathEscape(path.Join(user, defaultDevice, opt.Mountpoint)), + srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(rootURL), + pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), + } + f.features = (&fs.Features{ + CaseInsensitive: true, + CanHaveEmptyDirectories: true, + ReadMimeType: true, + WriteMimeType: true, + }).Fill(f) + + if user == "" || pass == "" { + return nil, errors.New("jottacloud needs user and password") + } + + f.srv.SetUserPass(opt.User, opt.Pass) + f.srv.SetErrorHandler(errorHandler) + + err = f.setEndpointURL(opt.Mountpoint) + if err != nil { + return nil, errors.Wrap(err, "couldn't get account info") + } + + if root != "" && !rootIsDir { + // Check to see if the root actually an existing file + remote := path.Base(root) + f.root = path.Dir(root) + if f.root == "." { + f.root = "" + } + _, err := f.NewObject(remote) + if err != nil { + if errors.Cause(err) == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile { + // File doesn't exist so return old f + f.root = root + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + + return f, nil +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + // Set info + err = o.setMetaData(info) + } else { + err = o.readMetaData() // reads info and meta, returning an error + } + if err != nil { + return nil, err + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// CreateDir makes a directory +func (f *Fs) CreateDir(path string) (jf *api.JottaFolder, err error) { + // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) + var resp *http.Response + opts := rest.Opts{ + Method: "POST", + Path: f.filePath(path), + Parameters: url.Values{}, + } + + opts.Parameters.Set("mkDir", "true") + + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(&opts, nil, &jf) + return shouldRetry(resp, err) + }) + if err != nil { + //fmt.Printf("...Error %v\n", err) + return nil, err + } + // fmt.Printf("...Id %q\n", *info.Id) + return jf, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + //fmt.Printf("List: %s\n", dir) + opts := rest.Opts{ + Method: "GET", + Path: f.filePath(dir), + } + + var resp *http.Response + var result api.JottaFolder + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(&opts, nil, &result) + return shouldRetry(resp, err) + }) + + if err != nil { + if apiErr, ok := err.(*api.Error); ok { + // does not exist + if apiErr.StatusCode == http.StatusNotFound { + return nil, fs.ErrorDirNotFound + } + } + return nil, errors.Wrap(err, "couldn't list files") + } + + if result.Deleted { + return nil, fs.ErrorDirNotFound + } + + for i := range result.Folders { + item := &result.Folders[i] + if item.Deleted { + continue + } + remote := path.Join(dir, restoreReservedChars(item.Name)) + d := fs.NewDir(remote, time.Time(item.ModifiedAt)) + entries = append(entries, d) + } + + for i := range result.Files { + item := &result.Files[i] + if item.Deleted || item.State != "COMPLETED" { + continue + } + remote := path.Join(dir, restoreReservedChars(item.Name)) + o, err := f.newObjectWithInfo(remote, item) + if err != nil { + continue + } + entries = append(entries, o) + } + //fmt.Printf("Entries: %+v\n", entries) + return entries, nil +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Used to create new objects +func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object) { + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + size: size, + modTime: modTime, + } + return o +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + o := f.createObject(src.Remote(), src.ModTime(), src.Size()) + return o, o.Update(in, src, options...) +} + +// mkParentDir makes the parent of the native path dirPath if +// necessary and any directories above that +func (f *Fs) mkParentDir(dirPath string) error { + // defer log.Trace(dirPath, "")("") + // chop off trailing / if it exists + if strings.HasSuffix(dirPath, "/") { + dirPath = dirPath[:len(dirPath)-1] + } + parent := path.Dir(dirPath) + if parent == "." { + parent = "" + } + return f.Mkdir(parent) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + _, err := f.CreateDir(dir) + return err +} + +// purgeCheck removes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(dir string, check bool) (err error) { + root := path.Join(f.root, dir) + if root == "" { + return errors.New("can't purge root directory") + } + + // check that the directory exists + entries, err := f.List(dir) + if err != nil { + return err + } + + if check { + if len(entries) != 0 { + return fs.ErrorDirectoryNotEmpty + } + } + + opts := rest.Opts{ + Method: "POST", + Path: f.filePath(dir), + Parameters: url.Values{}, + NoResponse: true, + } + + opts.Parameters.Set("dlDir", "true") + + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "rmdir failed") + } + + // TODO: Parse response? + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + return f.purgeCheck(dir, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + return f.purgeCheck("", false) +} + +// copyOrMoves copys or moves directories or files depending on the mthod parameter +func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err error) { + opts := rest.Opts{ + Method: "POST", + Path: src, + Parameters: url.Values{}, + } + + opts.Parameters.Set(method, "/"+path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, dest)))) + + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(&opts, nil, &info) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + + return info, nil +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantMove + } + + err := f.mkParentDir(remote) + if err != nil { + return nil, err + } + info, err := f.copyOrMove("cp", srcObj.filePath(), remote) + + if err != nil { + return nil, errors.Wrap(err, "copy failed") + } + + return f.newObjectWithInfo(remote, info) + //return f.newObjectWithInfo(remote, &result) +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + err := f.mkParentDir(remote) + if err != nil { + return nil, err + } + info, err := f.copyOrMove("mv", srcObj.filePath(), remote) + + if err != nil { + return nil, errors.Wrap(err, "move failed") + } + + return f.newObjectWithInfo(remote, info) + //return f.newObjectWithInfo(remote, result) +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := path.Join(srcFs.root, srcRemote) + dstPath := path.Join(f.root, dstRemote) + + // Refuse to move to or from the root + if srcPath == "" || dstPath == "" { + fs.Debugf(src, "DirMove error: Can't move root") + return errors.New("can't move root directory") + } + //fmt.Printf("Move src: %s (FullPath %s), dst: %s (FullPath: %s)\n", srcRemote, srcPath, dstRemote, dstPath) + + var err error + _, err = f.List(dstRemote) + if err == fs.ErrorDirNotFound { + // OK + } else if err != nil { + return err + } else { + return fs.ErrorDirExists + } + + _, err = f.copyOrMove("mvDir", path.Join(f.endpointURL, replaceReservedChars(srcPath))+"/", dstRemote) + + if err != nil { + return errors.Wrap(err, "moveDir failed") + } + return nil +} + +// About gets quota information +func (f *Fs) About() (*fs.Usage, error) { + info, err := f.getAccountInfo() + if err != nil { + return nil, err + } + + usage := &fs.Usage{ + Total: fs.NewUsageValue(info.Capacity), + Used: fs.NewUsageValue(info.Usage), + } + return usage, nil +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +// --------------------------------------------- + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the MD5 of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.MD5 { + return "", hash.ErrUnsupported + } + return o.md5, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return 0 + } + return o.size +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + return o.mimeType +} + +// setMetaData sets the metadata from info +func (o *Object) setMetaData(info *api.JottaFile) (err error) { + o.hasMetaData = true + o.size = int64(info.Size) + o.md5 = info.MD5 + o.mimeType = info.MimeType + o.modTime = time.Time(info.ModifiedAt) + return nil +} + +func (o *Object) readMetaData() (err error) { + if o.hasMetaData { + return nil + } + info, err := o.fs.readMetaDataForPath(o.remote) + if err != nil { + return err + } + return o.setMetaData(info) +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + return fs.ErrorCantSetModTime +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + fs.FixRangeOption(options, o.size) + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + Path: o.filePath(), + Parameters: url.Values{}, + Options: options, + } + + opts.Parameters.Set("mode", "bin") + + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + return resp.Body, err +} + +// Read the md5 of in returning a reader which will read the same contents +// +// The cleanup function should be called when out is finished with +// regardless of whether this function returned an error or not. +func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) { + // we need a MD5 + md5Hasher := md5.New() + // use the teeReader to write to the local file AND caclulate the MD5 while doing so + teeReader := io.TeeReader(in, md5Hasher) + + // nothing to clean up by default + cleanup = func() {} + + // don't cache small files on disk to reduce wear of the disk + if size > threshold { + var tempFile *os.File + + // create the cache file + tempFile, err = ioutil.TempFile("", cachePrefix) + if err != nil { + return + } + + _ = os.Remove(tempFile.Name()) // Delete the file - may not work on Windows + + // clean up the file after we are done downloading + cleanup = func() { + // the file should normally already be close, but just to make sure + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) // delete the cache file after we are done - may be deleted already + } + + // copy the ENTIRE file to disc and calculate the MD5 in the process + if _, err = io.Copy(tempFile, teeReader); err != nil { + return + } + // jump to the start of the local file so we can pass it along + if _, err = tempFile.Seek(0, 0); err != nil { + return + } + + // replace the already read source with a reader of our cached file + out = tempFile + } else { + // that's a small file, just read it into memory + var inData []byte + inData, err = ioutil.ReadAll(teeReader) + if err != nil { + return + } + + // set the reader to our read memory block + out = bytes.NewReader(inData) + } + return hex.EncodeToString(md5Hasher.Sum(nil)), out, cleanup, nil +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// If existing is set then it updates the object rather than creating a new one +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + size := src.Size() + md5String, err := src.Hash(hash.MD5) + if err != nil || md5String == "" { + // unwrap the accounting from the input, we use wrap to put it + // back on after the buffering + var wrap accounting.WrapFn + in, wrap = accounting.UnWrap(in) + var cleanup func() + md5String, in, cleanup, err = readMD5(in, size, int64(o.fs.opt.MD5MemoryThreshold)) + defer cleanup() + if err != nil { + return errors.Wrap(err, "failed to calculate MD5") + } + // Wrap the accounting back onto the stream + in = wrap(in) + } + + var resp *http.Response + var result api.JottaFile + opts := rest.Opts{ + Method: "POST", + Path: o.filePath(), + Body: in, + ContentType: fs.MimeType(src), + ContentLength: &size, + ExtraHeaders: make(map[string]string), + Parameters: url.Values{}, + } + + opts.ExtraHeaders["JMd5"] = md5String + opts.Parameters.Set("cphash", md5String) + opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10) + // opts.ExtraHeaders["JCreated"] = api.Time(src.ModTime()).String() + opts.ExtraHeaders["JModified"] = api.Time(src.ModTime()).String() + + // Parameters observed in other implementations + //opts.ExtraHeaders["X-Jfs-DeviceName"] = "Jotta" + //opts.ExtraHeaders["X-Jfs-Devicename-Base64"] = "" + //opts.ExtraHeaders["X-Jftp-Version"] = "2.4" this appears to be the current version + //opts.ExtraHeaders["jx_csid"] = "" + //opts.ExtraHeaders["jx_lisence"] = "" + + opts.Parameters.Set("umode", "nomultipart") + + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + resp, err = o.fs.srv.CallXML(&opts, nil, &result) + return shouldRetry(resp, err) + }) + if err != nil { + return err + } + + // TODO: Check returned Metadata? Timeout on big uploads? + return o.setMetaData(&result) +} + +// Remove an object +func (o *Object) Remove() error { + opts := rest.Opts{ + Method: "POST", + Path: o.filePath(), + Parameters: url.Values{}, + } + + opts.Parameters.Set("dl", "true") + + return o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.CallXML(&opts, nil, nil) + return shouldRetry(resp, err) + }) +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = (*Object)(nil) +) diff --git a/.rclone_repo/backend/jottacloud/jottacloud_internal_test.go b/.rclone_repo/backend/jottacloud/jottacloud_internal_test.go new file mode 100755 index 0000000..f5fe7b2 --- /dev/null +++ b/.rclone_repo/backend/jottacloud/jottacloud_internal_test.go @@ -0,0 +1,67 @@ +package jottacloud + +import ( + "crypto/md5" + "fmt" + "io" + "io/ioutil" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A test reader to return a test pattern of size +type testReader struct { + size int64 + c byte +} + +// Reader is the interface that wraps the basic Read method. +func (r *testReader) Read(p []byte) (n int, err error) { + for i := range p { + if r.size <= 0 { + return n, io.EOF + } + p[i] = r.c + r.c = (r.c + 1) % 253 + r.size-- + n++ + } + return +} + +func TestReadMD5(t *testing.T) { + // smoke test the reader + b, err := ioutil.ReadAll(&testReader{size: 10}) + require.NoError(t, err) + assert.Equal(t, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, b) + + // Check readMD5 for different size and threshold + for _, size := range []int64{0, 1024, 10 * 1024, 100 * 1024} { + t.Run(fmt.Sprintf("%d", size), func(t *testing.T) { + hasher := md5.New() + n, err := io.Copy(hasher, &testReader{size: size}) + require.NoError(t, err) + assert.Equal(t, n, size) + wantMD5 := fmt.Sprintf("%x", hasher.Sum(nil)) + for _, threshold := range []int64{512, 1024, 10 * 1024, 20 * 1024} { + t.Run(fmt.Sprintf("%d", threshold), func(t *testing.T) { + in := &testReader{size: size} + gotMD5, out, cleanup, err := readMD5(in, size, threshold) + defer cleanup() + require.NoError(t, err) + assert.Equal(t, wantMD5, gotMD5) + + // check md5hash of out + hasher := md5.New() + n, err := io.Copy(hasher, out) + require.NoError(t, err) + assert.Equal(t, n, size) + outMD5 := fmt.Sprintf("%x", hasher.Sum(nil)) + assert.Equal(t, wantMD5, outMD5) + }) + } + }) + } +} diff --git a/.rclone_repo/backend/jottacloud/jottacloud_test.go b/.rclone_repo/backend/jottacloud/jottacloud_test.go new file mode 100755 index 0000000..9bede91 --- /dev/null +++ b/.rclone_repo/backend/jottacloud/jottacloud_test.go @@ -0,0 +1,17 @@ +// Test Box filesystem interface +package jottacloud_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/jottacloud" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestJottacloud:", + NilObject: (*jottacloud.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/jottacloud/replace.go b/.rclone_repo/backend/jottacloud/replace.go new file mode 100755 index 0000000..c6622b3 --- /dev/null +++ b/.rclone_repo/backend/jottacloud/replace.go @@ -0,0 +1,84 @@ +/* +Translate file names for JottaCloud adapted from OneDrive + + +The following characters are JottaClous reserved characters, and can't +be used in JottaCloud folder and file names. + + jottacloud = "/" / "\" / "*" / "<" / ">" / "?" / "!" / "&" / ":" / ";" / "|" / "#" / "%" / """ / "'" / "." / "~" + + +*/ + +package jottacloud + +import ( + "regexp" + "strings" +) + +// charMap holds replacements for characters +// +// Onedrive has a restricted set of characters compared to other cloud +// storage systems, so we to map these to the FULLWIDTH unicode +// equivalents +// +// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS +var ( + charMap = map[rune]rune{ + '\\': '＼', // FULLWIDTH REVERSE SOLIDUS + '+': '＋', // FULLWIDTH PLUS SIGN + '*': '＊', // FULLWIDTH ASTERISK + '<': '＜', // FULLWIDTH LESS-THAN SIGN + '>': '＞', // FULLWIDTH GREATER-THAN SIGN + '?': '？', // FULLWIDTH QUESTION MARK + '!': '！', // FULLWIDTH EXCLAMATION MARK + '&': '＆', // FULLWIDTH AMPERSAND + ':': '：', // FULLWIDTH COLON + ';': '；', // FULLWIDTH SEMICOLON + '|': '｜', // FULLWIDTH VERTICAL LINE + '#': '＃', // FULLWIDTH NUMBER SIGN + '%': '％', // FULLWIDTH PERCENT SIGN + '"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved + '\'': '＇', // FULLWIDTH APOSTROPHE + '~': '～', // FULLWIDTH TILDE + ' ': '␠', // SYMBOL FOR SPACE + } + invCharMap map[rune]rune + fixStartingWithSpace = regexp.MustCompile(`(/|^) `) + fixEndingWithSpace = regexp.MustCompile(` (/|$)`) +) + +func init() { + // Create inverse charMap + invCharMap = make(map[rune]rune, len(charMap)) + for k, v := range charMap { + invCharMap[v] = k + } +} + +// replaceReservedChars takes a path and substitutes any reserved +// characters in it +func replaceReservedChars(in string) string { + // Filenames can't start with space + in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' '])) + // Filenames can't end with space + in = fixEndingWithSpace.ReplaceAllString(in, string(charMap[' '])+"$1") + return strings.Map(func(c rune) rune { + if replacement, ok := charMap[c]; ok && c != ' ' { + return replacement + } + return c + }, in) +} + +// restoreReservedChars takes a path and undoes any substitutions +// made by replaceReservedChars +func restoreReservedChars(in string) string { + return strings.Map(func(c rune) rune { + if replacement, ok := invCharMap[c]; ok { + return replacement + } + return c + }, in) +} diff --git a/.rclone_repo/backend/jottacloud/replace_test.go b/.rclone_repo/backend/jottacloud/replace_test.go new file mode 100755 index 0000000..38d1308 --- /dev/null +++ b/.rclone_repo/backend/jottacloud/replace_test.go @@ -0,0 +1,28 @@ +package jottacloud + +import "testing" + +func TestReplace(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"", ""}, + {"abc 123", "abc 123"}, + {`\+*<>?!&:;|#%"'~`, `＼＋＊＜＞？！＆：；｜＃％＂＇～`}, + {`\+*<>?!&:;|#%"'~\+*<>?!&:;|#%"'~`, `＼＋＊＜＞？！＆：；｜＃％＂＇～＼＋＊＜＞？！＆：；｜＃％＂＇～`}, + {" leading space", "␠leading space"}, + {"trailing space ", "trailing space␠"}, + {" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"}, + {"trailing space /trailing space /trailing space ", "trailing space␠/trailing space␠/trailing space␠"}, + } { + got := replaceReservedChars(test.in) + if got != test.out { + t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got) + } + got2 := restoreReservedChars(got) + if got2 != test.in { + t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2) + } + } +} diff --git a/.rclone_repo/backend/local/about_unix.go b/.rclone_repo/backend/local/about_unix.go new file mode 100755 index 0000000..745f2e5 --- /dev/null +++ b/.rclone_repo/backend/local/about_unix.go @@ -0,0 +1,29 @@ +// +build darwin dragonfly freebsd linux + +package local + +import ( + "syscall" + + "github.com/ncw/rclone/fs" + "github.com/pkg/errors" +) + +// About gets quota information +func (f *Fs) About() (*fs.Usage, error) { + var s syscall.Statfs_t + err := syscall.Statfs(f.root, &s) + if err != nil { + return nil, errors.Wrap(err, "failed to read disk usage") + } + bs := int64(s.Bsize) + usage := &fs.Usage{ + Total: fs.NewUsageValue(bs * int64(s.Blocks)), // quota of bytes that can be used + Used: fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), // bytes in use + Free: fs.NewUsageValue(bs * int64(s.Bavail)), // bytes which can be uploaded before reaching the quota + } + return usage, nil +} + +// check interface +var _ fs.Abouter = &Fs{} diff --git a/.rclone_repo/backend/local/about_windows.go b/.rclone_repo/backend/local/about_windows.go new file mode 100755 index 0000000..4c9dcec --- /dev/null +++ b/.rclone_repo/backend/local/about_windows.go @@ -0,0 +1,36 @@ +// +build windows + +package local + +import ( + "syscall" + "unsafe" + + "github.com/ncw/rclone/fs" + "github.com/pkg/errors" +) + +var getFreeDiskSpace = syscall.NewLazyDLL("kernel32.dll").NewProc("GetDiskFreeSpaceExW") + +// About gets quota information +func (f *Fs) About() (*fs.Usage, error) { + var available, total, free int64 + _, _, e1 := getFreeDiskSpace.Call( + uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(f.root))), + uintptr(unsafe.Pointer(&available)), // lpFreeBytesAvailable - for this user + uintptr(unsafe.Pointer(&total)), // lpTotalNumberOfBytes + uintptr(unsafe.Pointer(&free)), // lpTotalNumberOfFreeBytes + ) + if e1 != syscall.Errno(0) { + return nil, errors.Wrap(e1, "failed to read disk usage") + } + usage := &fs.Usage{ + Total: fs.NewUsageValue(total), // quota of bytes that can be used + Used: fs.NewUsageValue(total - free), // bytes in use + Free: fs.NewUsageValue(available), // bytes which can be uploaded before reaching the quota + } + return usage, nil +} + +// check interface +var _ fs.Abouter = &Fs{} diff --git a/.rclone_repo/backend/local/local.go b/.rclone_repo/backend/local/local.go new file mode 100755 index 0000000..26ae8ed --- /dev/null +++ b/.rclone_repo/backend/local/local.go @@ -0,0 +1,993 @@ +// Package local provides a filesystem interface +package local + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "regexp" + "runtime" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/readers" + "github.com/pkg/errors" +) + +// Constants +const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset + +// Register with Fs +func init() { + fsi := &fs.RegInfo{ + Name: "local", + Description: "Local Disk", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "nounc", + Help: "Disable UNC (long path names) conversion on Windows", + Examples: []fs.OptionExample{{ + Value: "true", + Help: "Disables long file names", + }}, + }, { + Name: "copy_links", + Help: "Follow symlinks and copy the pointed to item.", + Default: false, + NoPrefix: true, + ShortOpt: "L", + Advanced: true, + }, { + Name: "skip_links", + Help: "Don't warn about skipped symlinks.", + Default: false, + NoPrefix: true, + Advanced: true, + }, { + Name: "no_unicode_normalization", + Help: "Don't apply unicode normalization to paths and filenames", + Default: false, + Advanced: true, + }, { + Name: "no_check_updated", + Help: "Don't check to see if the files change during upload", + Default: false, + Advanced: true, + }, { + Name: "one_file_system", + Help: "Don't cross filesystem boundaries (unix/macOS only).", + Default: false, + NoPrefix: true, + ShortOpt: "x", + Advanced: true, + }}, + } + fs.Register(fsi) +} + +// Options defines the configuration for this backend +type Options struct { + FollowSymlinks bool `config:"copy_links"` + SkipSymlinks bool `config:"skip_links"` + NoUTFNorm bool `config:"no_unicode_normalization"` + NoCheckUpdated bool `config:"no_check_updated"` + NoUNC bool `config:"nounc"` + OneFileSystem bool `config:"one_file_system"` +} + +// Fs represents a local filesystem rooted at root +type Fs struct { + name string // the name of the remote + root string // The root directory (OS path) + opt Options // parsed config options + features *fs.Features // optional features + dev uint64 // device number of root node + precisionOk sync.Once // Whether we need to read the precision + precision time.Duration // precision of local filesystem + wmu sync.Mutex // used for locking access to 'warned'. + warned map[string]struct{} // whether we have warned about this string + // do os.Lstat or os.Stat + lstat func(name string) (os.FileInfo, error) + dirNames *mapper // directory name mapping + objectHashesMu sync.Mutex // global lock for Object.hashes +} + +// Object represents a local filesystem object +type Object struct { + fs *Fs // The Fs this object is part of + remote string // The remote path - properly UTF-8 encoded - for rclone + path string // The local path - may not be properly UTF-8 encoded - for OS + size int64 // file metadata - always present + mode os.FileMode + modTime time.Time + hashes map[hash.Type]string // Hashes +} + +// ------------------------------------------------------------ + +// NewFs constructs an Fs from the path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + if opt.NoUTFNorm { + fs.Errorf(nil, "The --local-no-unicode-normalization flag is deprecated and will be removed") + } + + f := &Fs{ + name: name, + opt: *opt, + warned: make(map[string]struct{}), + dev: devUnset, + lstat: os.Lstat, + dirNames: newMapper(), + } + f.root = f.cleanPath(root) + f.features = (&fs.Features{ + CaseInsensitive: f.caseInsensitive(), + CanHaveEmptyDirectories: true, + }).Fill(f) + if opt.FollowSymlinks { + f.lstat = os.Stat + } + + // Check to see if this points to a file + fi, err := f.lstat(f.root) + if err == nil { + f.dev = readDevice(fi, f.opt.OneFileSystem) + } + if err == nil && fi.Mode().IsRegular() { + // It is a file, so use the parent as the root + f.root = filepath.Dir(f.root) + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + return f, nil +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("Local file system at %s", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// caseInsenstive returns whether the remote is case insensitive or not +func (f *Fs) caseInsensitive() bool { + // FIXME not entirely accurate since you can have case + // sensitive Fses on darwin and case insenstive Fses on linux. + // Should probably check but that would involve creating a + // file in the remote to be most accurate which probably isn't + // desirable. + return runtime.GOOS == "windows" || runtime.GOOS == "darwin" +} + +// newObject makes a half completed Object +// +// if dstPath is empty then it is made from remote +func (f *Fs) newObject(remote, dstPath string) *Object { + if dstPath == "" { + dstPath = f.cleanPath(filepath.Join(f.root, remote)) + } + remote = f.cleanRemote(remote) + return &Object{ + fs: f, + remote: remote, + path: dstPath, + } +} + +// Return an Object from a path +// +// May return nil if an error occurred +func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Object, error) { + o := f.newObject(remote, dstPath) + if info != nil { + o.setMetadata(info) + } else { + err := o.lstat() + if err != nil { + if os.IsNotExist(err) { + return nil, fs.ErrorObjectNotFound + } + if os.IsPermission(err) { + return nil, fs.ErrorPermissionDenied + } + return nil, err + } + } + if o.mode.IsDir() { + return nil, errors.Wrapf(fs.ErrorNotAFile, "%q", remote) + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, "", nil) +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + dir = f.dirNames.Load(dir) + fsDirPath := f.cleanPath(filepath.Join(f.root, dir)) + remote := f.cleanRemote(dir) + _, err = os.Stat(fsDirPath) + if err != nil { + return nil, fs.ErrorDirNotFound + } + + fd, err := os.Open(fsDirPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to open directory %q", dir) + } + defer func() { + cerr := fd.Close() + if cerr != nil && err == nil { + err = errors.Wrapf(cerr, "failed to close directory %q:", dir) + } + }() + + for { + fis, err := fd.Readdir(1024) + if err == io.EOF && len(fis) == 0 { + break + } + if err != nil { + return nil, errors.Wrapf(err, "failed to read directory %q", dir) + } + + for _, fi := range fis { + name := fi.Name() + mode := fi.Mode() + newRemote := path.Join(remote, name) + newPath := filepath.Join(fsDirPath, name) + // Follow symlinks if required + if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 { + fi, err = os.Stat(newPath) + if err != nil { + return nil, err + } + mode = fi.Mode() + } + if fi.IsDir() { + // Ignore directories which are symlinks. These are junction points under windows which + // are kind of a souped up symlink. Unix doesn't have directories which are symlinks. + if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi, f.opt.OneFileSystem) { + d := fs.NewDir(f.dirNames.Save(newRemote, f.cleanRemote(newRemote)), fi.ModTime()) + entries = append(entries, d) + } + } else { + fso, err := f.newObjectWithInfo(newRemote, newPath, fi) + if err != nil { + return nil, err + } + if fso.Storable() { + entries = append(entries, fso) + } + } + } + } + return entries, nil +} + +// cleanRemote makes string a valid UTF-8 string for remote strings. +// +// Any invalid UTF-8 characters will be replaced with utf8.RuneError +// It also normalises the UTF-8 and converts the slashes if necessary. +func (f *Fs) cleanRemote(name string) string { + if !utf8.ValidString(name) { + f.wmu.Lock() + if _, ok := f.warned[name]; !ok { + fs.Logf(f, "Replacing invalid UTF-8 characters in %q", name) + f.warned[name] = struct{}{} + } + f.wmu.Unlock() + name = string([]rune(name)) + } + name = filepath.ToSlash(name) + return name +} + +// mapper maps raw to cleaned directory names +type mapper struct { + mu sync.RWMutex // mutex to protect the below + m map[string]string // map of un-normalised directory names +} + +func newMapper() *mapper { + return &mapper{ + m: make(map[string]string), + } +} + +// Lookup a directory name to make a local name (reverses +// cleanDirName) +// +// FIXME this is temporary before we make a proper Directory object +func (m *mapper) Load(in string) string { + m.mu.RLock() + out, ok := m.m[in] + m.mu.RUnlock() + if ok { + return out + } + return in +} + +// Cleans a directory name recording if it needed to be altered +// +// FIXME this is temporary before we make a proper Directory object +func (m *mapper) Save(in, out string) string { + if in != out { + m.mu.Lock() + m.m[out] = in + m.mu.Unlock() + } + return out +} + +// Put the Object to the local filesystem +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + // Temporary Object under construction - info filled in by Update() + o := f.newObject(remote, "") + err := o.Update(in, src, options...) + if err != nil { + return nil, err + } + return o, nil +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// Mkdir creates the directory if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + // FIXME: https://github.com/syncthing/syncthing/blob/master/lib/osutil/mkdirall_windows.go + root := f.cleanPath(filepath.Join(f.root, dir)) + err := os.MkdirAll(root, 0777) + if err != nil { + return err + } + if dir == "" { + fi, err := f.lstat(root) + if err != nil { + return err + } + f.dev = readDevice(fi, f.opt.OneFileSystem) + } + return nil +} + +// Rmdir removes the directory +// +// If it isn't empty it will return an error +func (f *Fs) Rmdir(dir string) error { + root := f.cleanPath(filepath.Join(f.root, dir)) + return os.Remove(root) +} + +// Precision of the file system +func (f *Fs) Precision() (precision time.Duration) { + f.precisionOk.Do(func() { + f.precision = f.readPrecision() + }) + return f.precision +} + +// Read the precision +func (f *Fs) readPrecision() (precision time.Duration) { + // Default precision of 1s + precision = time.Second + + // Create temporary file and test it + fd, err := ioutil.TempFile("", "rclone") + if err != nil { + // If failed return 1s + // fmt.Println("Failed to create temp file", err) + return time.Second + } + path := fd.Name() + // fmt.Println("Created temp file", path) + err = fd.Close() + if err != nil { + return time.Second + } + + // Delete it on return + defer func() { + // fmt.Println("Remove temp file") + _ = os.Remove(path) // ignore error + }() + + // Find the minimum duration we can detect + for duration := time.Duration(1); duration < time.Second; duration *= 10 { + // Current time with delta + t := time.Unix(time.Now().Unix(), int64(duration)) + err := os.Chtimes(path, t, t) + if err != nil { + // fmt.Println("Failed to Chtimes", err) + break + } + + // Read the actual time back + fi, err := os.Stat(path) + if err != nil { + // fmt.Println("Failed to Stat", err) + break + } + + // If it matches - have found the precision + // fmt.Println("compare", fi.ModTime(), t) + if fi.ModTime().Equal(t) { + // fmt.Println("Precision detected as", duration) + return duration + } + } + return +} + +// Purge deletes all the files and directories +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + fi, err := f.lstat(f.root) + if err != nil { + return err + } + if !fi.Mode().IsDir() { + return errors.Errorf("can't purge non directory: %q", f.root) + } + return os.RemoveAll(f.root) +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // Temporary Object under construction + dstObj := f.newObject(remote, "") + + // Check it is a file if it exists + err := dstObj.lstat() + if os.IsNotExist(err) { + // OK + } else if err != nil { + return nil, err + } else if !dstObj.mode.IsRegular() { + // It isn't a file + return nil, errors.New("can't move file onto non-file") + } + + // Create destination + err = dstObj.mkdirAll() + if err != nil { + return nil, err + } + + // Do the move + err = os.Rename(srcObj.path, dstObj.path) + if os.IsNotExist(err) { + // race condition, source was deleted in the meantime + return nil, err + } else if os.IsPermission(err) { + // not enough rights to write to dst + return nil, err + } else if err != nil { + // not quite clear, but probably trying to move a file across file system + // boundaries. Copying might still work. + fs.Debugf(src, "Can't move: %v: trying copy", err) + return nil, fs.ErrorCantMove + } + + // Update the info + err = dstObj.lstat() + if err != nil { + return nil, err + } + + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := f.cleanPath(filepath.Join(srcFs.root, srcRemote)) + dstPath := f.cleanPath(filepath.Join(f.root, dstRemote)) + + // Check if destination exists + _, err := os.Lstat(dstPath) + if !os.IsNotExist(err) { + return fs.ErrorDirExists + } + + // Create parent of destination + dstParentPath := filepath.Dir(dstPath) + err = os.MkdirAll(dstParentPath, 0777) + if err != nil { + return err + } + + // Do the move + err = os.Rename(srcPath, dstPath) + if os.IsNotExist(err) { + // race condition, source was deleted in the meantime + return err + } else if os.IsPermission(err) { + // not enough rights to write to dst + return err + } else if err != nil { + // not quite clear, but probably trying to move directory across file system + // boundaries. Copying might still work. + fs.Debugf(src, "Can't move dir: %v: trying copy", err) + return fs.ErrorCantDirMove + } + return nil +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Supported +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the requested hash of a file as a lowercase hex string +func (o *Object) Hash(r hash.Type) (string, error) { + // Check that the underlying file hasn't changed + oldtime := o.modTime + oldsize := o.size + err := o.lstat() + if err != nil { + return "", errors.Wrap(err, "hash: failed to stat") + } + + o.fs.objectHashesMu.Lock() + hashes := o.hashes + o.fs.objectHashesMu.Unlock() + + if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil { + in, err := os.Open(o.path) + if err != nil { + return "", errors.Wrap(err, "hash: failed to open") + } + hashes, err = hash.Stream(in) + closeErr := in.Close() + if err != nil { + return "", errors.Wrap(err, "hash: failed to read") + } + if closeErr != nil { + return "", errors.Wrap(closeErr, "hash: failed to close") + } + o.fs.objectHashesMu.Lock() + o.hashes = hashes + o.fs.objectHashesMu.Unlock() + } + return hashes[r], nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.size +} + +// ModTime returns the modification time of the object +func (o *Object) ModTime() time.Time { + return o.modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + err := os.Chtimes(o.path, modTime, modTime) + if err != nil { + return err + } + // Re-read metadata + return o.lstat() +} + +// Storable returns a boolean showing if this object is storable +func (o *Object) Storable() bool { + // Check for control characters in the remote name and show non storable + for _, c := range o.Remote() { + if c >= 0x00 && c < 0x20 || c == 0x7F { + fs.Logf(o.fs, "Can't store file with control characters: %q", o.Remote()) + return false + } + } + mode := o.mode + if mode&os.ModeSymlink != 0 { + if !o.fs.opt.SkipSymlinks { + fs.Logf(o, "Can't follow symlink without -L/--copy-links") + } + return false + } else if mode&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 { + fs.Logf(o, "Can't transfer non file/directory") + return false + } else if mode&os.ModeDir != 0 { + // fs.Debugf(o, "Skipping directory") + return false + } + return true +} + +// localOpenFile wraps an io.ReadCloser and updates the md5sum of the +// object that is read +type localOpenFile struct { + o *Object // object that is open + in io.ReadCloser // handle we are wrapping + hash *hash.MultiHasher // currently accumulating hashes + fd *os.File // file object reference +} + +// Read bytes from the object - see io.Reader +func (file *localOpenFile) Read(p []byte) (n int, err error) { + if !file.o.fs.opt.NoCheckUpdated { + // Check if file has the same size and modTime + fi, err := file.fd.Stat() + if err != nil { + return 0, errors.Wrap(err, "can't read status of source file while transferring") + } + if file.o.size != fi.Size() { + return 0, errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", file.o.size, fi.Size()) + } + if !file.o.modTime.Equal(fi.ModTime()) { + return 0, errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", file.o.modTime, fi.ModTime()) + } + } + + n, err = file.in.Read(p) + if n > 0 { + // Hash routines never return an error + _, _ = file.hash.Write(p[:n]) + } + return +} + +// Close the object and update the hashes +func (file *localOpenFile) Close() (err error) { + err = file.in.Close() + if err == nil { + if file.hash.Size() == file.o.Size() { + file.o.fs.objectHashesMu.Lock() + file.o.hashes = file.hash.Sums() + file.o.fs.objectHashesMu.Unlock() + } + } + return err +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + var offset, limit int64 = 0, -1 + hashes := hash.Supported + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + case *fs.RangeOption: + offset, limit = x.Decode(o.size) + case *fs.HashesOption: + hashes = x.Hashes + default: + if option.Mandatory() { + fs.Logf(o, "Unsupported mandatory option: %v", option) + } + } + } + + fd, err := os.Open(o.path) + if err != nil { + return + } + wrappedFd := readers.NewLimitedReadCloser(fd, limit) + if offset != 0 { + // seek the object + _, err = fd.Seek(offset, io.SeekStart) + // don't attempt to make checksums + return wrappedFd, err + } + hash, err := hash.NewMultiHasherTypes(hashes) + if err != nil { + return nil, err + } + // Update the md5sum as we go along + in = &localOpenFile{ + o: o, + in: wrappedFd, + hash: hash, + fd: fd, + } + return in, nil +} + +// mkdirAll makes all the directories needed to store the object +func (o *Object) mkdirAll() error { + dir := filepath.Dir(o.path) + return os.MkdirAll(dir, 0777) +} + +// Update the object from in with modTime and size +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + hashes := hash.Supported + for _, option := range options { + switch x := option.(type) { + case *fs.HashesOption: + hashes = x.Hashes + } + } + + err := o.mkdirAll() + if err != nil { + return err + } + + out, err := os.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return err + } + + // Calculate the hash of the object we are reading as we go along + hash, err := hash.NewMultiHasherTypes(hashes) + if err != nil { + return err + } + in = io.TeeReader(in, hash) + + _, err = io.Copy(out, in) + closeErr := out.Close() + if err == nil { + err = closeErr + } + if err != nil { + fs.Logf(o, "Removing partially written file on error: %v", err) + if removeErr := os.Remove(o.path); removeErr != nil { + fs.Errorf(o, "Failed to remove partially written file: %v", removeErr) + } + return err + } + + // All successful so update the hashes + o.fs.objectHashesMu.Lock() + o.hashes = hash.Sums() + o.fs.objectHashesMu.Unlock() + + // Set the mtime + err = o.SetModTime(src.ModTime()) + if err != nil { + return err + } + + // ReRead info now that we have finished + return o.lstat() +} + +// setMetadata sets the file info from the os.FileInfo passed in +func (o *Object) setMetadata(info os.FileInfo) { + // Don't overwrite the info if we don't need to + // this avoids upsetting the race detector + if o.size != info.Size() { + o.size = info.Size() + } + if !o.modTime.Equal(info.ModTime()) { + o.modTime = info.ModTime() + } + if o.mode != info.Mode() { + o.mode = info.Mode() + } +} + +// Stat a Object into info +func (o *Object) lstat() error { + info, err := o.fs.lstat(o.path) + if err == nil { + o.setMetadata(info) + } + return err +} + +// Remove an object +func (o *Object) Remove() error { + return remove(o.path) +} + +// cleanPathFragment cleans an OS path fragment which is part of a +// bigger path and not necessarily absolute +func cleanPathFragment(s string) string { + if s == "" { + return s + } + s = filepath.Clean(s) + if runtime.GOOS == "windows" { + s = strings.Replace(s, `/`, `\`, -1) + } + return s +} + +// cleanPath cleans and makes absolute the path passed in and returns +// an OS path. +// +// The input might be in OS form or rclone form or a mixture, but the +// output is in OS form. +// +// On windows it makes the path UNC also and replaces any characters +// Windows can't deal with with their replacements. +func (f *Fs) cleanPath(s string) string { + s = cleanPathFragment(s) + if runtime.GOOS == "windows" { + if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") { + s2, err := filepath.Abs(s) + if err == nil { + s = s2 + } + } + if !f.opt.NoUNC { + // Convert to UNC + s = uncPath(s) + } + s = cleanWindowsName(f, s) + } else { + if !filepath.IsAbs(s) { + s2, err := filepath.Abs(s) + if err == nil { + s = s2 + } + } + } + return s +} + +// Pattern to match a windows absolute path: "c:\" and similar +var isAbsWinDrive = regexp.MustCompile(`^[a-zA-Z]\:\\`) + +// uncPath converts an absolute Windows path +// to a UNC long path. +func uncPath(s string) string { + // UNC can NOT use "/", so convert all to "\" + s = strings.Replace(s, `/`, `\`, -1) + + // If prefix is "\\", we already have a UNC path or server. + if strings.HasPrefix(s, `\\`) { + // If already long path, just keep it + if strings.HasPrefix(s, `\\?\`) { + return s + } + + // Trim "\\" from path and add UNC prefix. + return `\\?\UNC\` + strings.TrimPrefix(s, `\\`) + } + if isAbsWinDrive.MatchString(s) { + return `\\?\` + s + } + return s +} + +// cleanWindowsName will clean invalid Windows characters replacing them with _ +func cleanWindowsName(f *Fs, name string) string { + original := name + var name2 string + if strings.HasPrefix(name, `\\?\`) { + name2 = `\\?\` + name = strings.TrimPrefix(name, `\\?\`) + } + if strings.HasPrefix(name, `//?/`) { + name2 = `//?/` + name = strings.TrimPrefix(name, `//?/`) + } + // Colon is allowed as part of a drive name X:\ + colonAt := strings.Index(name, ":") + if colonAt > 0 && colonAt < 3 && len(name) > colonAt+1 { + // Copy to name2, which is unfiltered + name2 += name[0 : colonAt+1] + name = name[colonAt+1:] + } + + name2 += strings.Map(func(r rune) rune { + switch r { + case '<', '>', '"', '|', '?', '*', ':': + return '_' + } + return r + }, name) + + if name2 != original && f != nil { + f.wmu.Lock() + if _, ok := f.warned[name]; !ok { + fs.Logf(f, "Replacing invalid characters in %q to %q", name, name2) + f.warned[name] = struct{}{} + } + f.wmu.Unlock() + } + return name2 +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.Purger = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.Mover = &Fs{} + _ fs.DirMover = &Fs{} + _ fs.Object = &Object{} +) diff --git a/.rclone_repo/backend/local/local_internal_test.go b/.rclone_repo/backend/local/local_internal_test.go new file mode 100755 index 0000000..d69e7cf --- /dev/null +++ b/.rclone_repo/backend/local/local_internal_test.go @@ -0,0 +1,74 @@ +package local + +import ( + "os" + "path" + "testing" + "time" + + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/fstest" + "github.com/ncw/rclone/lib/readers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMain drives the tests +func TestMain(m *testing.M) { + fstest.TestMain(m) +} + +func TestMapper(t *testing.T) { + m := newMapper() + assert.Equal(t, m.m, map[string]string{}) + assert.Equal(t, "potato", m.Save("potato", "potato")) + assert.Equal(t, m.m, map[string]string{}) + assert.Equal(t, "-r'áö", m.Save("-r?'a´o¨", "-r'áö")) + assert.Equal(t, m.m, map[string]string{ + "-r'áö": "-r?'a´o¨", + }) + assert.Equal(t, "potato", m.Load("potato")) + assert.Equal(t, "-r?'a´o¨", m.Load("-r'áö")) +} + +// Test copy with source file that's updating +func TestUpdatingCheck(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + filePath := "sub dir/local test" + r.WriteFile(filePath, "content", time.Now()) + + fd, err := os.Open(path.Join(r.LocalName, filePath)) + if err != nil { + t.Fatalf("failed opening file %q: %v", filePath, err) + } + + fi, err := fd.Stat() + require.NoError(t, err) + o := &Object{size: fi.Size(), modTime: fi.ModTime(), fs: &Fs{}} + wrappedFd := readers.NewLimitedReadCloser(fd, -1) + hash, err := hash.NewMultiHasherTypes(hash.Supported) + require.NoError(t, err) + in := localOpenFile{ + o: o, + in: wrappedFd, + hash: hash, + fd: fd, + } + + buf := make([]byte, 1) + _, err = in.Read(buf) + require.NoError(t, err) + + r.WriteFile(filePath, "content updated", time.Now()) + _, err = in.Read(buf) + require.Errorf(t, err, "can't copy - source file is being updated") + + // turn the checking off and try again + in.o.fs.opt.NoCheckUpdated = true + + r.WriteFile(filePath, "content updated", time.Now()) + _, err = in.Read(buf) + require.NoError(t, err) + +} diff --git a/.rclone_repo/backend/local/local_test.go b/.rclone_repo/backend/local/local_test.go new file mode 100755 index 0000000..81f9f6e --- /dev/null +++ b/.rclone_repo/backend/local/local_test.go @@ -0,0 +1,17 @@ +// Test Local filesystem interface +package local_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/local" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "", + NilObject: (*local.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/local/read_device_other.go b/.rclone_repo/backend/local/read_device_other.go new file mode 100755 index 0000000..c3fc4f4 --- /dev/null +++ b/.rclone_repo/backend/local/read_device_other.go @@ -0,0 +1,13 @@ +// Device reading functions + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris + +package local + +import "os" + +// readDevice turns a valid os.FileInfo into a device number, +// returning devUnset if it fails. +func readDevice(fi os.FileInfo, oneFileSystem bool) uint64 { + return devUnset +} diff --git a/.rclone_repo/backend/local/read_device_unix.go b/.rclone_repo/backend/local/read_device_unix.go new file mode 100755 index 0000000..1b2b0c5 --- /dev/null +++ b/.rclone_repo/backend/local/read_device_unix.go @@ -0,0 +1,26 @@ +// Device reading functions + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package local + +import ( + "os" + "syscall" + + "github.com/ncw/rclone/fs" +) + +// readDevice turns a valid os.FileInfo into a device number, +// returning devUnset if it fails. +func readDevice(fi os.FileInfo, oneFileSystem bool) uint64 { + if !oneFileSystem { + return devUnset + } + statT, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + fs.Debugf(fi.Name(), "Type assertion fi.Sys().(*syscall.Stat_t) failed from: %#v", fi.Sys()) + return devUnset + } + return uint64(statT.Dev) +} diff --git a/.rclone_repo/backend/local/remove_other.go b/.rclone_repo/backend/local/remove_other.go new file mode 100755 index 0000000..760e2cf --- /dev/null +++ b/.rclone_repo/backend/local/remove_other.go @@ -0,0 +1,10 @@ +//+build !windows + +package local + +import "os" + +// Removes name, retrying on a sharing violation +func remove(name string) error { + return os.Remove(name) +} diff --git a/.rclone_repo/backend/local/remove_test.go b/.rclone_repo/backend/local/remove_test.go new file mode 100755 index 0000000..b2e34cd --- /dev/null +++ b/.rclone_repo/backend/local/remove_test.go @@ -0,0 +1,50 @@ +package local + +import ( + "io/ioutil" + "os" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Check we can remove an open file +func TestRemove(t *testing.T) { + fd, err := ioutil.TempFile("", "rclone-remove-test") + require.NoError(t, err) + name := fd.Name() + defer func() { + _ = os.Remove(name) + }() + + exists := func() bool { + _, err := os.Stat(name) + if err == nil { + return true + } else if os.IsNotExist(err) { + return false + } + require.NoError(t, err) + return false + } + + assert.True(t, exists()) + // close the file in the background + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + time.Sleep(250 * time.Millisecond) + require.NoError(t, fd.Close()) + }() + // delete the open file + err = remove(name) + require.NoError(t, err) + // check it no longer exists + assert.False(t, exists()) + // wait for background close + wg.Wait() +} diff --git a/.rclone_repo/backend/local/remove_windows.go b/.rclone_repo/backend/local/remove_windows.go new file mode 100755 index 0000000..bf28b2c --- /dev/null +++ b/.rclone_repo/backend/local/remove_windows.go @@ -0,0 +1,38 @@ +//+build windows + +package local + +import ( + "os" + "syscall" + "time" + + "github.com/ncw/rclone/fs" +) + +const ( + ERROR_SHARING_VIOLATION syscall.Errno = 32 +) + +// Removes name, retrying on a sharing violation +func remove(name string) (err error) { + const maxTries = 10 + var sleepTime = 1 * time.Millisecond + for i := 0; i < maxTries; i++ { + err = os.Remove(name) + if err == nil { + break + } + pathErr, ok := err.(*os.PathError) + if !ok { + break + } + if pathErr.Err != ERROR_SHARING_VIOLATION { + break + } + fs.Logf(name, "Remove detected sharing violation - retry %d/%d sleeping %v", i+1, maxTries, sleepTime) + time.Sleep(sleepTime) + sleepTime <<= 1 + } + return err +} diff --git a/.rclone_repo/backend/local/tests_test.go b/.rclone_repo/backend/local/tests_test.go new file mode 100755 index 0000000..c8be144 --- /dev/null +++ b/.rclone_repo/backend/local/tests_test.go @@ -0,0 +1,91 @@ +package local + +import ( + "testing" +) + +var uncTestPaths = []string{ + "C:\\Ba*d\\P|a?t\\Windows\\Folder", + "C:/Ba*d/P|a?t/Windows\\Folder", + "C:\\Windows\\Folder", + "\\\\?\\C:\\Windows\\Folder", + "//?/C:/Windows/Folder", + "\\\\?\\UNC\\server\\share\\Desktop", + "\\\\?\\unC\\server\\share\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path", + "\\\\server\\share\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path", + "C:\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path", + "C:\\AbsoluteToRoot\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path", + "\\\\server\\share\\Desktop", + "\\\\?\\UNC\\\\share\\folder\\Desktop", + "\\\\server\\share", +} + +var uncTestPathsResults = []string{ + `\\?\C:\Ba*d\P|a?t\Windows\Folder`, + `\\?\C:\Ba*d\P|a?t\Windows\Folder`, + `\\?\C:\Windows\Folder`, + `\\?\C:\Windows\Folder`, + `\\?\C:\Windows\Folder`, + `\\?\UNC\server\share\Desktop`, + `\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`, + `\\?\UNC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`, + `\\?\C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`, + `\\?\C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`, + `\\?\UNC\server\share\Desktop`, + `\\?\UNC\\share\folder\Desktop`, + `\\?\UNC\server\share`, +} + +// Test that UNC paths are converted. +func TestUncPaths(t *testing.T) { + for i, p := range uncTestPaths { + unc := uncPath(p) + if unc != uncTestPathsResults[i] { + t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i]) + } + // Test we don't add more. + unc = uncPath(unc) + if unc != uncTestPathsResults[i] { + t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i]) + } + } +} + +var utf8Tests = [][2]string{ + {"ABC", "ABC"}, + {string([]byte{0x80}), "�"}, + {string([]byte{'a', 0x80, 'b'}), "a�b"}, +} + +func TestCleanRemote(t *testing.T) { + f := &Fs{} + f.warned = make(map[string]struct{}) + for _, test := range utf8Tests { + got := f.cleanRemote(test[0]) + expect := test[1] + if got != expect { + t.Fatalf("got %q, expected %q", got, expect) + } + } +} + +// Test Windows character replacements +var testsWindows = [][2]string{ + {`c:\temp`, `c:\temp`}, + {`\\?\UNC\theserver\dir\file.txt`, `\\?\UNC\theserver\dir\file.txt`}, + {`//?/UNC/theserver/dir\file.txt`, `//?/UNC/theserver/dir\file.txt`}, + {"c:/temp", "c:/temp"}, + {"/temp/file.txt", "/temp/file.txt"}, + {`!\"#¤%&/()=;:*^?+-`, "!\\_#¤%&/()=;__^_+-"}, + {`<>"|?*:&\<>"|?*:&\<>"|?*:&`, "_______&\\_______&\\_______&"}, +} + +func TestCleanWindows(t *testing.T) { + for _, test := range testsWindows { + got := cleanWindowsName(nil, test[0]) + expect := test[1] + if got != expect { + t.Fatalf("got %q, expected %q", got, expect) + } + } +} diff --git a/.rclone_repo/backend/mega/mega.go b/.rclone_repo/backend/mega/mega.go new file mode 100755 index 0000000..a754441 --- /dev/null +++ b/.rclone_repo/backend/mega/mega.go @@ -0,0 +1,1161 @@ +// Package mega provides an interface to the Mega +// object storage system. +package mega + +/* +Open questions +* Does mega support a content hash - what exactly are the mega hashes? +* Can mega support setting modification times? + +Improvements: +* Uploads could be done in parallel +* Downloads would be more efficient done in one go +* Uploads would be more efficient with bigger chunks +* Looks like mega can support server side copy, but it isn't implemented in go-mega +* Upload can set modtime... - set as int64_t - can set ctime and mtime? +*/ + +import ( + "fmt" + "io" + "path" + "strings" + "sync" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/pacer" + "github.com/ncw/rclone/lib/readers" + "github.com/pkg/errors" + mega "github.com/t3rm1n4l/go-mega" +) + +const ( + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + eventWaitTime = 500 * time.Millisecond + decayConstant = 2 // bigger for slower decay, exponential +) + +var ( + megaCacheMu sync.Mutex // mutex for the below + megaCache = map[string]*mega.Mega{} // cache logged in Mega's by user +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "mega", + Description: "Mega", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "user", + Help: "User name", + Required: true, + }, { + Name: "pass", + Help: "Password.", + Required: true, + IsPassword: true, + }, { + Name: "debug", + Help: "Output more debug from Mega.", + Default: false, + Advanced: true, + }, { + Name: "hard_delete", + Help: "Delete files permanently rather than putting them into the trash.", + Default: false, + Advanced: true, + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + User string `config:"user"` + Pass string `config:"pass"` + Debug bool `config:"debug"` + HardDelete bool `config:"hard_delete"` +} + +// Fs represents a remote mega +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed config options + features *fs.Features // optional features + srv *mega.Mega // the connection to the server + pacer *pacer.Pacer // pacer for API calls + rootNodeMu sync.Mutex // mutex for _rootNode + _rootNode *mega.Node // root node - call findRoot to use this + mkdirMu sync.Mutex // used to serialize calls to mkdir / rmdir +} + +// Object describes a mega object +// +// Will definitely have info but maybe not meta +// +// Normally rclone would just store an ID here but go-mega and mega.nz +// expect you to build an entire tree of all the objects in memory. +// In this case we just store a pointer to the object. +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + info *mega.Node // pointer to the mega node +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("mega root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// parsePath parses an mega 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// shouldRetry returns a boolean as to whether this err deserves to be +// retried. It returns the err as a convenience +func shouldRetry(err error) (bool, error) { + // Let the mega library handle the low level retries + return false, err + /* + switch errors.Cause(err) { + case mega.EAGAIN, mega.ERATELIMIT, mega.ETEMPUNAVAIL: + return true, err + } + return fserrors.ShouldRetry(err), err + */ +} + +// readMetaDataForPath reads the metadata from the path +func (f *Fs) readMetaDataForPath(remote string) (info *mega.Node, err error) { + rootNode, err := f.findRoot(false) + if err != nil { + return nil, err + } + return f.findObject(rootNode, remote) +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + if opt.Pass != "" { + var err error + opt.Pass, err = obscure.Reveal(opt.Pass) + if err != nil { + return nil, errors.Wrap(err, "couldn't decrypt password") + } + } + + // cache *mega.Mega on username so we can re-use and share + // them between remotes. They are expensive to make as they + // contain all the objects and sharing the objects makes the + // move code easier as we don't have to worry about mixing + // them up between different remotes. + megaCacheMu.Lock() + defer megaCacheMu.Unlock() + srv := megaCache[opt.User] + if srv == nil { + srv = mega.New().SetClient(fshttp.NewClient(fs.Config)) + srv.SetRetries(fs.Config.LowLevelRetries) // let mega do the low level retries + srv.SetLogger(func(format string, v ...interface{}) { + fs.Infof("*go-mega*", format, v...) + }) + if opt.Debug { + srv.SetDebugger(func(format string, v ...interface{}) { + fs.Debugf("*go-mega*", format, v...) + }) + } + + err := srv.Login(opt.User, opt.Pass) + if err != nil { + return nil, errors.Wrap(err, "couldn't login") + } + megaCache[opt.User] = srv + } + + root = parsePath(root) + f := &Fs{ + name: name, + root: root, + opt: *opt, + srv: srv, + pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), + } + f.features = (&fs.Features{ + DuplicateFiles: true, + CanHaveEmptyDirectories: true, + }).Fill(f) + + // Find the root node and check if it is a file or not + _, err = f.findRoot(false) + switch err { + case nil: + // root node found and is a directory + case fs.ErrorDirNotFound: + // root node not found, so can't be a file + case fs.ErrorIsFile: + // root node is a file so point to parent directory + root = path.Dir(root) + if root == "." { + root = "" + } + f.root = root + return f, err + } + return f, nil +} + +// splitNodePath splits nodePath into / separated parts, returning nil if it +// should refer to the root +func splitNodePath(nodePath string) (parts []string) { + nodePath = path.Clean(nodePath) + parts = strings.Split(nodePath, "/") + if len(parts) == 1 && (parts[0] == "." || parts[0] == "/") { + return nil + } + return parts +} + +// findNode looks up the node for the path of the name given from the root given +// +// It returns mega.ENOENT if it wasn't found +func (f *Fs) findNode(rootNode *mega.Node, nodePath string) (*mega.Node, error) { + parts := splitNodePath(nodePath) + if parts == nil { + return rootNode, nil + } + nodes, err := f.srv.FS.PathLookup(rootNode, parts) + if err != nil { + return nil, err + } + return nodes[len(nodes)-1], nil +} + +// findDir finds the directory rooted from the node passed in +func (f *Fs) findDir(rootNode *mega.Node, dir string) (node *mega.Node, err error) { + node, err = f.findNode(rootNode, dir) + if err == mega.ENOENT { + return nil, fs.ErrorDirNotFound + } else if err == nil && node.GetType() == mega.FILE { + return nil, fs.ErrorIsFile + } + return node, err +} + +// findObject looks up the node for the object of the name given +func (f *Fs) findObject(rootNode *mega.Node, file string) (node *mega.Node, err error) { + node, err = f.findNode(rootNode, file) + if err == mega.ENOENT { + return nil, fs.ErrorObjectNotFound + } else if err == nil && node.GetType() != mega.FILE { + return nil, fs.ErrorNotAFile + } + return node, err +} + +// lookupDir looks up the node for the directory of the name given +// +// if create is true it tries to create the root directory if not found +func (f *Fs) lookupDir(dir string) (*mega.Node, error) { + rootNode, err := f.findRoot(false) + if err != nil { + return nil, err + } + return f.findDir(rootNode, dir) +} + +// lookupParentDir finds the parent node for the remote passed in +func (f *Fs) lookupParentDir(remote string) (dirNode *mega.Node, leaf string, err error) { + parent, leaf := path.Split(remote) + dirNode, err = f.lookupDir(parent) + return dirNode, leaf, err +} + +// mkdir makes the directory and any parent directories for the +// directory of the name given +func (f *Fs) mkdir(rootNode *mega.Node, dir string) (node *mega.Node, err error) { + f.mkdirMu.Lock() + defer f.mkdirMu.Unlock() + + parts := splitNodePath(dir) + if parts == nil { + return rootNode, nil + } + var i int + // look up until we find a directory which exists + for i = 0; i <= len(parts); i++ { + var nodes []*mega.Node + nodes, err = f.srv.FS.PathLookup(rootNode, parts[:len(parts)-i]) + if err == nil { + if len(nodes) == 0 { + node = rootNode + } else { + node = nodes[len(nodes)-1] + } + break + } + if err != mega.ENOENT { + return nil, errors.Wrap(err, "mkdir lookup failed") + } + } + if err != nil { + return nil, errors.Wrap(err, "internal error: mkdir called with non existent root node") + } + // i is number of directories to create (may be 0) + // node is directory to create them from + for _, name := range parts[len(parts)-i:] { + // create directory called name in node + err = f.pacer.Call(func() (bool, error) { + node, err = f.srv.CreateDir(name, node) + return shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "mkdir create node failed") + } + } + return node, nil +} + +// mkdirParent creates the parent directory of remote +func (f *Fs) mkdirParent(remote string) (dirNode *mega.Node, leaf string, err error) { + rootNode, err := f.findRoot(true) + if err != nil { + return nil, "", err + } + parent, leaf := path.Split(remote) + dirNode, err = f.mkdir(rootNode, parent) + return dirNode, leaf, err +} + +// findRoot looks up the root directory node and returns it. +// +// if create is true it tries to create the root directory if not found +func (f *Fs) findRoot(create bool) (*mega.Node, error) { + f.rootNodeMu.Lock() + defer f.rootNodeMu.Unlock() + + // Check if we haven't found it already + if f._rootNode != nil { + return f._rootNode, nil + } + + // Check for pre-existing root + absRoot := f.srv.FS.GetRoot() + node, err := f.findDir(absRoot, f.root) + //log.Printf("findRoot findDir %p %v", node, err) + if err == nil { + f._rootNode = node + return node, nil + } + if !create || err != fs.ErrorDirNotFound { + return nil, err + } + + //..not found so create the root directory + f._rootNode, err = f.mkdir(absRoot, f.root) + return f._rootNode, err +} + +// clearRoot unsets the root directory +func (f *Fs) clearRoot() { + f.rootNodeMu.Lock() + f._rootNode = nil + f.rootNodeMu.Unlock() + //log.Printf("cleared root directory") +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *mega.Node) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + // Set info + err = o.setMetaData(info) + } else { + err = o.readMetaData() // reads info and meta, returning an error + } + if err != nil { + return nil, err + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// list the objects into the function supplied +// +// If directories is set it only sends directories +// User function to process a File item from listAll +// +// Should return true to finish processing +type listFn func(*mega.Node) bool + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +func (f *Fs) list(dir *mega.Node, fn listFn) (found bool, err error) { + nodes, err := f.srv.FS.GetChildren(dir) + if err != nil { + return false, errors.Wrapf(err, "list failed") + } + for _, item := range nodes { + if fn(item) { + found = true + break + } + } + return +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + dirNode, err := f.lookupDir(dir) + if err != nil { + return nil, err + } + var iErr error + _, err = f.list(dirNode, func(info *mega.Node) bool { + remote := path.Join(dir, info.GetName()) + switch info.GetType() { + case mega.FOLDER, mega.ROOT, mega.INBOX, mega.TRASH: + d := fs.NewDir(remote, info.GetTimeStamp()).SetID(info.GetHash()) + entries = append(entries, d) + case mega.FILE: + o, err := f.newObjectWithInfo(remote, info) + if err != nil { + iErr = err + return true + } + entries = append(entries, o) + } + return false + }) + if err != nil { + return nil, err + } + if iErr != nil { + return nil, iErr + } + return entries, nil +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Returns the dirNode, obect, leaf and error +// +// Used to create new objects +func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) { + dirNode, leaf, err = f.mkdirParent(remote) + if err != nil { + return nil, nil, leaf, err + } + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + } + return o, dirNode, leaf, nil +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +// PutUnchecked uploads the object +// +// This will create a duplicate if we upload a new file without +// checking to see if there is one already - use Put() for that. +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil) + switch err { + case nil: + return exisitingObj, exisitingObj.Update(in, src, options...) + case fs.ErrorObjectNotFound: + // Not found so create it + return f.PutUnchecked(in, src) + default: + return nil, err + } +} + +// PutUnchecked the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +// PutUnchecked uploads the object +// +// This will create a duplicate if we upload a new file without +// checking to see if there is one already - use Put() for that. +func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime() + + o, _, _, err := f.createObject(remote, modTime, size) + if err != nil { + return nil, err + } + return o, o.Update(in, src, options...) +} + +// Mkdir creates the directory if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + rootNode, err := f.findRoot(true) + if err != nil { + return err + } + _, err = f.mkdir(rootNode, dir) + return errors.Wrap(err, "Mkdir failed") +} + +// deleteNode removes a file or directory, observing useTrash +func (f *Fs) deleteNode(node *mega.Node) (err error) { + err = f.pacer.Call(func() (bool, error) { + err = f.srv.Delete(node, f.opt.HardDelete) + return shouldRetry(err) + }) + return err +} + +// purgeCheck removes the directory dir, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(dir string, check bool) error { + f.mkdirMu.Lock() + defer f.mkdirMu.Unlock() + + rootNode, err := f.findRoot(false) + if err != nil { + return err + } + dirNode, err := f.findDir(rootNode, dir) + if err != nil { + return err + } + + if check { + children, err := f.srv.FS.GetChildren(dirNode) + if err != nil { + return errors.Wrap(err, "purgeCheck GetChildren failed") + } + if len(children) > 0 { + return fs.ErrorDirectoryNotEmpty + } + } + + waitEvent := f.srv.WaitEventsStart() + + err = f.deleteNode(dirNode) + if err != nil { + return errors.Wrap(err, "delete directory node failed") + } + + // Remove the root node if we just deleted it + if dirNode == rootNode { + f.clearRoot() + } + + f.srv.WaitEvents(waitEvent, eventWaitTime) + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + return f.purgeCheck(dir, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + return f.purgeCheck("", false) +} + +// move a file or folder (srcFs, srcRemote, info) to (f, dstRemote) +// +// info will be updates +func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node) (err error) { + var ( + dstFs = f + srcDirNode, dstDirNode *mega.Node + srcParent, dstParent string + srcLeaf, dstLeaf string + ) + + if dstRemote != "" { + // lookup or create the destination parent directory + dstDirNode, dstLeaf, err = dstFs.mkdirParent(dstRemote) + } else { + // find or create the parent of the root directory + absRoot := dstFs.srv.FS.GetRoot() + dstParent, dstLeaf = path.Split(dstFs.root) + dstDirNode, err = dstFs.mkdir(absRoot, dstParent) + } + if err != nil { + return errors.Wrap(err, "server side move failed to make dst parent dir") + } + + if srcRemote != "" { + // lookup the existing parent directory + srcDirNode, srcLeaf, err = srcFs.lookupParentDir(srcRemote) + } else { + // lookup the existing root parent + absRoot := srcFs.srv.FS.GetRoot() + srcParent, srcLeaf = path.Split(srcFs.root) + srcDirNode, err = f.findDir(absRoot, srcParent) + } + if err != nil { + return errors.Wrap(err, "server side move failed to lookup src parent dir") + } + + // move the object into its new directory if required + if srcDirNode != dstDirNode && srcDirNode.GetHash() != dstDirNode.GetHash() { + //log.Printf("move src %p %q dst %p %q", srcDirNode, srcDirNode.GetName(), dstDirNode, dstDirNode.GetName()) + err = f.pacer.Call(func() (bool, error) { + err = f.srv.Move(info, dstDirNode) + return shouldRetry(err) + }) + if err != nil { + return errors.Wrap(err, "server side move failed") + } + } + + waitEvent := f.srv.WaitEventsStart() + + // rename the object if required + if srcLeaf != dstLeaf { + //log.Printf("rename %q to %q", srcLeaf, dstLeaf) + err = f.pacer.Call(func() (bool, error) { + err = f.srv.Rename(info, dstLeaf) + return shouldRetry(err) + }) + if err != nil { + return errors.Wrap(err, "server side rename failed") + } + } + + f.srv.WaitEvents(waitEvent, eventWaitTime) + + return nil +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + dstFs := f + + //log.Printf("Move %q -> %q", src.Remote(), remote) + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // Do the move + err := f.move(remote, srcObj.fs, srcObj.remote, srcObj.info) + if err != nil { + return nil, err + } + + // Create a destination object + dstObj := &Object{ + fs: dstFs, + remote: remote, + info: srcObj.info, + } + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + dstFs := f + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + + // find the source + info, err := srcFs.lookupDir(srcRemote) + if err != nil { + return err + } + + // check the destination doesn't exist + _, err = dstFs.lookupDir(dstRemote) + if err == nil { + return fs.ErrorDirExists + } else if err != fs.ErrorDirNotFound { + return errors.Wrap(err, "DirMove error while checking dest directory") + } + + // Do the move + err = f.move(dstRemote, srcFs, srcRemote, info) + if err != nil { + return err + } + + // Clear src if it was the root + if srcRemote == "" { + srcFs.clearRoot() + } + + return nil +} + +// DirCacheFlush an optional interface to flush internal directory cache +func (f *Fs) DirCacheFlush() { + // f.dirCache.ResetRoot() + // FIXME Flush the mega somehow? +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.None) +} + +// PublicLink generates a public link to the remote path (usually readable by anyone) +func (f *Fs) PublicLink(remote string) (link string, err error) { + root, err := f.findRoot(false) + if err != nil { + return "", errors.Wrap(err, "PublicLink failed to find root node") + } + node, err := f.findNode(root, remote) + if err != nil { + return "", errors.Wrap(err, "PublicLink failed to find path") + } + link, err = f.srv.Link(node, true) + if err != nil { + return "", errors.Wrap(err, "PublicLink failed to create link") + } + return link, nil +} + +// MergeDirs merges the contents of all the directories passed +// in into the first one and rmdirs the other directories. +func (f *Fs) MergeDirs(dirs []fs.Directory) error { + if len(dirs) < 2 { + return nil + } + // find dst directory + dstDir := dirs[0] + dstDirNode := f.srv.FS.HashLookup(dstDir.ID()) + if dstDirNode == nil { + return errors.Errorf("MergeDirs failed to find node for: %v", dstDir) + } + for _, srcDir := range dirs[1:] { + // find src directory + srcDirNode := f.srv.FS.HashLookup(srcDir.ID()) + if srcDirNode == nil { + return errors.Errorf("MergeDirs failed to find node for: %v", srcDir) + } + + // list the the objects + infos := []*mega.Node{} + _, err := f.list(srcDirNode, func(info *mega.Node) bool { + infos = append(infos, info) + return false + }) + if err != nil { + return errors.Wrapf(err, "MergeDirs list failed on %v", srcDir) + } + // move them into place + for _, info := range infos { + fs.Infof(srcDir, "merging %q", info.GetName()) + err = f.pacer.Call(func() (bool, error) { + err = f.srv.Move(info, dstDirNode) + return shouldRetry(err) + }) + if err != nil { + return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.GetName(), srcDir) + } + } + // rmdir (into trash) the now empty source directory + fs.Infof(srcDir, "removing empty directory") + err = f.deleteNode(srcDirNode) + if err != nil { + return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir) + } + } + return nil +} + +// About gets quota information +func (f *Fs) About() (*fs.Usage, error) { + var q mega.QuotaResp + var err error + err = f.pacer.Call(func() (bool, error) { + q, err = f.srv.GetQuota() + return shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to get Mega Quota") + } + usage := &fs.Usage{ + Total: fs.NewUsageValue(int64(q.Mstrg)), // quota of bytes that can be used + Used: fs.NewUsageValue(int64(q.Cstrg)), // bytes in use + Free: fs.NewUsageValue(int64(q.Mstrg - q.Cstrg)), // bytes which can be uploaded before reaching the quota + } + return usage, nil +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the hashes of an object +func (o *Object) Hash(t hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.info.GetSize() +} + +// setMetaData sets the metadata from info +func (o *Object) setMetaData(info *mega.Node) (err error) { + if info.GetType() != mega.FILE { + return fs.ErrorNotAFile + } + o.info = info + return nil +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData() (err error) { + if o.info != nil { + return nil + } + info, err := o.fs.readMetaDataForPath(o.remote) + if err != nil { + if err == fs.ErrorDirNotFound { + err = fs.ErrorObjectNotFound + } + return err + } + return o.setMetaData(info) +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + return o.info.GetTimeStamp() +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + return fs.ErrorCantSetModTime +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// openObject represents a download in progress +type openObject struct { + mu sync.Mutex + o *Object + d *mega.Download + id int + skip int64 + chunk []byte + closed bool +} + +// get the next chunk +func (oo *openObject) getChunk() (err error) { + if oo.id >= oo.d.Chunks() { + return io.EOF + } + var chunk []byte + err = oo.o.fs.pacer.Call(func() (bool, error) { + chunk, err = oo.d.DownloadChunk(oo.id) + return shouldRetry(err) + }) + if err != nil { + return err + } + oo.id++ + oo.chunk = chunk + return nil +} + +// Read reads up to len(p) bytes into p. +func (oo *openObject) Read(p []byte) (n int, err error) { + oo.mu.Lock() + defer oo.mu.Unlock() + if oo.closed { + return 0, errors.New("read on closed file") + } + // Skip data at the start if requested + for oo.skip > 0 { + _, size, err := oo.d.ChunkLocation(oo.id) + if err != nil { + return 0, err + } + if oo.skip < int64(size) { + break + } + oo.id++ + oo.skip -= int64(size) + } + if len(oo.chunk) == 0 { + err = oo.getChunk() + if err != nil { + return 0, err + } + if oo.skip > 0 { + oo.chunk = oo.chunk[oo.skip:] + oo.skip = 0 + } + } + n = copy(p, oo.chunk) + oo.chunk = oo.chunk[n:] + return n, nil +} + +// Close closed the file - MAC errors are reported here +func (oo *openObject) Close() (err error) { + oo.mu.Lock() + defer oo.mu.Unlock() + if oo.closed { + return nil + } + err = oo.o.fs.pacer.Call(func() (bool, error) { + err = oo.d.Finish() + return shouldRetry(err) + }) + if err != nil { + return errors.Wrap(err, "failed to finish download") + } + oo.closed = true + return nil +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + var offset, limit int64 = 0, -1 + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + case *fs.RangeOption: + offset, limit = x.Decode(o.Size()) + default: + if option.Mandatory() { + fs.Logf(o, "Unsupported mandatory option: %v", option) + } + } + } + + var d *mega.Download + err = o.fs.pacer.Call(func() (bool, error) { + d, err = o.fs.srv.NewDownload(o.info) + return shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "open download file failed") + } + + oo := &openObject{ + o: o, + d: d, + skip: offset, + } + + return readers.NewLimitedReadCloser(oo, limit), nil +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// If existing is set then it updates the object rather than creating a new one +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + size := src.Size() + //modTime := src.ModTime() + remote := o.Remote() + + // Create the parent directory + dirNode, leaf, err := o.fs.mkdirParent(remote) + if err != nil { + return errors.Wrap(err, "update make parent dir failed") + } + + var u *mega.Upload + err = o.fs.pacer.Call(func() (bool, error) { + u, err = o.fs.srv.NewUpload(dirNode, leaf, size) + return shouldRetry(err) + }) + if err != nil { + return errors.Wrap(err, "upload file failed to create session") + } + + // Upload the chunks + // FIXME do this in parallel + for id := 0; id < u.Chunks(); id++ { + _, chunkSize, err := u.ChunkLocation(id) + if err != nil { + return errors.Wrap(err, "upload failed to read chunk location") + } + chunk := make([]byte, chunkSize) + _, err = io.ReadFull(in, chunk) + if err != nil { + return errors.Wrap(err, "upload failed to read data") + } + + err = o.fs.pacer.Call(func() (bool, error) { + err = u.UploadChunk(id, chunk) + return shouldRetry(err) + }) + if err != nil { + return errors.Wrap(err, "upload file failed to upload chunk") + } + } + + // Finish the upload + var info *mega.Node + err = o.fs.pacer.Call(func() (bool, error) { + info, err = u.Finish() + return shouldRetry(err) + }) + if err != nil { + return errors.Wrap(err, "failed to finish upload") + } + + // If the upload succeded and the original object existed, then delete it + if o.info != nil { + err = o.fs.deleteNode(o.info) + if err != nil { + return errors.Wrap(err, "upload failed to remove old version") + } + o.info = nil + } + + return o.setMetaData(info) +} + +// Remove an object +func (o *Object) Remove() error { + err := o.fs.deleteNode(o.info) + if err != nil { + return errors.Wrap(err, "Remove object failed") + } + return nil +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.info.GetHash() +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.PutUncheckeder = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.PublicLinker = (*Fs)(nil) + _ fs.MergeDirser = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.IDer = (*Object)(nil) +) diff --git a/.rclone_repo/backend/mega/mega_test.go b/.rclone_repo/backend/mega/mega_test.go new file mode 100755 index 0000000..37d0798 --- /dev/null +++ b/.rclone_repo/backend/mega/mega_test.go @@ -0,0 +1,17 @@ +// Test Mega filesystem interface +package mega_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/mega" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestMega:", + NilObject: (*mega.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/onedrive/api/types.go b/.rclone_repo/backend/onedrive/api/types.go new file mode 100755 index 0000000..6491f26 --- /dev/null +++ b/.rclone_repo/backend/onedrive/api/types.go @@ -0,0 +1,359 @@ +// Types passed and returned to and from the API + +package api + +import ( + "strings" + "time" +) + +const ( + timeFormat = `"` + time.RFC3339 + `"` +) + +// Error is returned from one drive when things go wrong +type Error struct { + ErrorInfo struct { + Code string `json:"code"` + Message string `json:"message"` + InnerError struct { + Code string `json:"code"` + } `json:"innererror"` + } `json:"error"` +} + +// Error returns a string for the error and statistifes the error interface +func (e *Error) Error() string { + out := e.ErrorInfo.Code + if e.ErrorInfo.InnerError.Code != "" { + out += ": " + e.ErrorInfo.InnerError.Code + } + out += ": " + e.ErrorInfo.Message + return out +} + +// Check Error statisfies the error interface +var _ error = (*Error)(nil) + +// Identity represents an identity of an actor. For example, and actor +// can be a user, device, or application. +type Identity struct { + DisplayName string `json:"displayName"` + ID string `json:"id"` +} + +// IdentitySet is a keyed collection of Identity objects. It is used +// to represent a set of identities associated with various events for +// an item, such as created by or last modified by. +type IdentitySet struct { + User Identity `json:"user"` + Application Identity `json:"application"` + Device Identity `json:"device"` +} + +// Quota groups storage space quota-related information on OneDrive into a single structure. +type Quota struct { + Total int64 `json:"total"` + Used int64 `json:"used"` + Remaining int64 `json:"remaining"` + Deleted int64 `json:"deleted"` + State string `json:"state"` // normal | nearing | critical | exceeded +} + +// Drive is a representation of a drive resource +type Drive struct { + ID string `json:"id"` + DriveType string `json:"driveType"` + Owner IdentitySet `json:"owner"` + Quota Quota `json:"quota"` +} + +// Timestamp represents represents date and time information for the +// OneDrive API, by using ISO 8601 and is always in UTC time. +type Timestamp time.Time + +// MarshalJSON turns a Timestamp into JSON (in UTC) +func (t *Timestamp) MarshalJSON() (out []byte, err error) { + timeString := (*time.Time)(t).UTC().Format(timeFormat) + return []byte(timeString), nil +} + +// UnmarshalJSON turns JSON into a Timestamp +func (t *Timestamp) UnmarshalJSON(data []byte) error { + newT, err := time.Parse(timeFormat, string(data)) + if err != nil { + return err + } + *t = Timestamp(newT) + return nil +} + +// ItemReference groups data needed to reference a OneDrive item +// across the service into a single structure. +type ItemReference struct { + DriveID string `json:"driveId"` // Unique identifier for the Drive that contains the item. Read-only. + ID string `json:"id"` // Unique identifier for the item. Read/Write. + Path string `json:"path"` // Path that used to navigate to the item. Read/Write. +} + +// RemoteItemFacet groups data needed to reference a OneDrive remote item +type RemoteItemFacet struct { + ID string `json:"id"` // The unique identifier of the item within the remote Drive. Read-only. + Name string `json:"name"` // The name of the item (filename and extension). Read-write. + CreatedBy IdentitySet `json:"createdBy"` // Identity of the user, device, and application which created the item. Read-only. + LastModifiedBy IdentitySet `json:"lastModifiedBy"` // Identity of the user, device, and application which last modified the item. Read-only. + CreatedDateTime Timestamp `json:"createdDateTime"` // Date and time of item creation. Read-only. + LastModifiedDateTime Timestamp `json:"lastModifiedDateTime"` // Date and time the item was last modified. Read-only. + Folder *FolderFacet `json:"folder"` // Folder metadata, if the item is a folder. Read-only. + File *FileFacet `json:"file"` // File metadata, if the item is a file. Read-only. + FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write. + ParentReference *ItemReference `json:"parentReference"` // Parent information, if the item has a parent. Read-write. + Size int64 `json:"size"` // Size of the item in bytes. Read-only. + WebURL string `json:"webUrl"` // URL that displays the resource in the browser. Read-only. +} + +// FolderFacet groups folder-related data on OneDrive into a single structure +type FolderFacet struct { + ChildCount int64 `json:"childCount"` // Number of children contained immediately within this container. +} + +// HashesType groups different types of hashes into a single structure, for an item on OneDrive. +type HashesType struct { + Sha1Hash string `json:"sha1Hash"` // hex encoded SHA1 hash for the contents of the file (if available) + Crc32Hash string `json:"crc32Hash"` // hex encoded CRC32 value of the file (if available) + QuickXorHash string `json:"quickXorHash"` // base64 encoded QuickXorHash value of the file (if available) +} + +// FileFacet groups file-related data on OneDrive into a single structure. +type FileFacet struct { + MimeType string `json:"mimeType"` // The MIME type for the file. This is determined by logic on the server and might not be the value provided when the file was uploaded. + Hashes HashesType `json:"hashes"` // Hashes of the file's binary content, if available. +} + +// FileSystemInfoFacet contains properties that are reported by the +// device's local file system for the local version of an item. This +// facet can be used to specify the last modified date or created date +// of the item as it was on the local device. +type FileSystemInfoFacet struct { + CreatedDateTime Timestamp `json:"createdDateTime"` // The UTC date and time the file was created on a client. + LastModifiedDateTime Timestamp `json:"lastModifiedDateTime"` // The UTC date and time the file was last modified on a client. +} + +// DeletedFacet indicates that the item on OneDrive has been +// deleted. In this version of the API, the presence (non-null) of the +// facet value indicates that the file was deleted. A null (or +// missing) value indicates that the file is not deleted. +type DeletedFacet struct { +} + +// Item represents metadata for an item in OneDrive +type Item struct { + ID string `json:"id"` // The unique identifier of the item within the Drive. Read-only. + Name string `json:"name"` // The name of the item (filename and extension). Read-write. + ETag string `json:"eTag"` // eTag for the entire item (metadata + content). Read-only. + CTag string `json:"cTag"` // An eTag for the content of the item. This eTag is not changed if only the metadata is changed. Read-only. + CreatedBy IdentitySet `json:"createdBy"` // Identity of the user, device, and application which created the item. Read-only. + LastModifiedBy IdentitySet `json:"lastModifiedBy"` // Identity of the user, device, and application which last modified the item. Read-only. + CreatedDateTime Timestamp `json:"createdDateTime"` // Date and time of item creation. Read-only. + LastModifiedDateTime Timestamp `json:"lastModifiedDateTime"` // Date and time the item was last modified. Read-only. + Size int64 `json:"size"` // Size of the item in bytes. Read-only. + ParentReference *ItemReference `json:"parentReference"` // Parent information, if the item has a parent. Read-write. + WebURL string `json:"webUrl"` // URL that displays the resource in the browser. Read-only. + Description string `json:"description"` // Provide a user-visible description of the item. Read-write. + Folder *FolderFacet `json:"folder"` // Folder metadata, if the item is a folder. Read-only. + File *FileFacet `json:"file"` // File metadata, if the item is a file. Read-only. + RemoteItem *RemoteItemFacet `json:"remoteItem"` // Remote Item metadata, if the item is a remote shared item. Read-only. + FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write. + // Image *ImageFacet `json:"image"` // Image metadata, if the item is an image. Read-only. + // Photo *PhotoFacet `json:"photo"` // Photo metadata, if the item is a photo. Read-only. + // Audio *AudioFacet `json:"audio"` // Audio metadata, if the item is an audio file. Read-only. + // Video *VideoFacet `json:"video"` // Video metadata, if the item is a video. Read-only. + // Location *LocationFacet `json:"location"` // Location metadata, if the item has location data. Read-only. + Deleted *DeletedFacet `json:"deleted"` // Information about the deleted state of the item. Read-only. +} + +// ViewDeltaResponse is the response to the view delta method +type ViewDeltaResponse struct { + Value []Item `json:"value"` // An array of Item objects which have been created, modified, or deleted. + NextLink string `json:"@odata.nextLink"` // A URL to retrieve the next available page of changes. + DeltaLink string `json:"@odata.deltaLink"` // A URL returned instead of @odata.nextLink after all current changes have been returned. Used to read the next set of changes in the future. + DeltaToken string `json:"@delta.token"` // A token value that can be used in the query string on manually-crafted calls to view.delta. Not needed if you're using nextLink and deltaLink. +} + +// ListChildrenResponse is the response to the list children method +type ListChildrenResponse struct { + Value []Item `json:"value"` // An array of Item objects + NextLink string `json:"@odata.nextLink"` // A URL to retrieve the next available page of items. +} + +// CreateItemRequest is the request to create an item object +type CreateItemRequest struct { + Name string `json:"name"` // Name of the folder to be created. + Folder FolderFacet `json:"folder"` // Empty Folder facet to indicate that folder is the type of resource to be created. + ConflictBehavior string `json:"@name.conflictBehavior"` // Determines what to do if an item with a matching name already exists in this folder. Accepted values are: rename, replace, and fail (the default). +} + +// SetFileSystemInfo is used to Update an object's FileSystemInfo. +type SetFileSystemInfo struct { + FileSystemInfo FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write. +} + +// CreateUploadRequest is used by CreateUploadSession to set the dates correctly +type CreateUploadRequest struct { + Item SetFileSystemInfo `json:"item"` +} + +// CreateUploadResponse is the response from creating an upload session +type CreateUploadResponse struct { + UploadURL string `json:"uploadUrl"` // "https://sn3302.up.1drv.com/up/fe6987415ace7X4e1eF866337", + ExpirationDateTime Timestamp `json:"expirationDateTime"` // "2015-01-29T09:21:55.523Z", + NextExpectedRanges []string `json:"nextExpectedRanges"` // ["0-"] +} + +// UploadFragmentResponse is the response from uploading a fragment +type UploadFragmentResponse struct { + ExpirationDateTime Timestamp `json:"expirationDateTime"` // "2015-01-29T09:21:55.523Z", + NextExpectedRanges []string `json:"nextExpectedRanges"` // ["0-"] +} + +// CopyItemRequest is the request to copy an item object +// +// Note: The parentReference should include either an id or path but +// not both. If both are included, they need to reference the same +// item or an error will occur. +type CopyItemRequest struct { + ParentReference ItemReference `json:"parentReference"` // Reference to the parent item the copy will be created in. + Name *string `json:"name"` // Optional The new name for the copy. If this isn't provided, the same name will be used as the original. +} + +// MoveItemRequest is the request to copy an item object +// +// Note: The parentReference should include either an id or path but +// not both. If both are included, they need to reference the same +// item or an error will occur. +type MoveItemRequest struct { + ParentReference *ItemReference `json:"parentReference,omitempty"` // Reference to the destination parent directory + Name string `json:"name,omitempty"` // Optional The new name for the file. If this isn't provided, the same name will be used as the original. + FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo,omitempty"` // File system information on client. Read-write. +} + +// AsyncOperationStatus provides information on the status of a asynchronous job progress. +// +// The following API calls return AsyncOperationStatus resources: +// +// Copy Item +// Upload From URL +type AsyncOperationStatus struct { + Operation string `json:"operation"` // The type of job being run. + PercentageComplete float64 `json:"percentageComplete"` // An float value between 0 and 100 that indicates the percentage complete. + Status string `json:"status"` // A string value that maps to an enumeration of possible values about the status of the job. "notStarted | inProgress | completed | updating | failed | deletePending | deleteFailed | waiting" +} + +// GetID returns a normalized ID of the item +// If DriveID is known it will be prefixed to the ID with # seperator +func (i *Item) GetID() string { + if i.IsRemote() && i.RemoteItem.ID != "" { + return i.RemoteItem.ParentReference.DriveID + "#" + i.RemoteItem.ID + } else if i.ParentReference != nil && strings.Index(i.ID, "#") == -1 { + return i.ParentReference.DriveID + "#" + i.ID + } + return i.ID +} + +// GetDriveID returns a normalized ParentReferance of the item +func (i *Item) GetDriveID() string { + return i.GetParentReferance().DriveID +} + +// GetName returns a normalized Name of the item +func (i *Item) GetName() string { + if i.IsRemote() && i.RemoteItem.Name != "" { + return i.RemoteItem.Name + } + return i.Name +} + +// GetFolder returns a normalized Folder of the item +func (i *Item) GetFolder() *FolderFacet { + if i.IsRemote() && i.RemoteItem.Folder != nil { + return i.RemoteItem.Folder + } + return i.Folder +} + +// GetFile returns a normalized File of the item +func (i *Item) GetFile() *FileFacet { + if i.IsRemote() && i.RemoteItem.File != nil { + return i.RemoteItem.File + } + return i.File +} + +// GetFileSystemInfo returns a normalized FileSystemInfo of the item +func (i *Item) GetFileSystemInfo() *FileSystemInfoFacet { + if i.IsRemote() && i.RemoteItem.FileSystemInfo != nil { + return i.RemoteItem.FileSystemInfo + } + return i.FileSystemInfo +} + +// GetSize returns a normalized Size of the item +func (i *Item) GetSize() int64 { + if i.IsRemote() && i.RemoteItem.Size != 0 { + return i.RemoteItem.Size + } + return i.Size +} + +// GetWebURL returns a normalized WebURL of the item +func (i *Item) GetWebURL() string { + if i.IsRemote() && i.RemoteItem.WebURL != "" { + return i.RemoteItem.WebURL + } + return i.WebURL +} + +// GetCreatedBy returns a normalized CreatedBy of the item +func (i *Item) GetCreatedBy() IdentitySet { + if i.IsRemote() && i.RemoteItem.CreatedBy != (IdentitySet{}) { + return i.RemoteItem.CreatedBy + } + return i.CreatedBy +} + +// GetLastModifiedBy returns a normalized LastModifiedBy of the item +func (i *Item) GetLastModifiedBy() IdentitySet { + if i.IsRemote() && i.RemoteItem.LastModifiedBy != (IdentitySet{}) { + return i.RemoteItem.LastModifiedBy + } + return i.LastModifiedBy +} + +// GetCreatedDateTime returns a normalized CreatedDateTime of the item +func (i *Item) GetCreatedDateTime() Timestamp { + if i.IsRemote() && i.RemoteItem.CreatedDateTime != (Timestamp{}) { + return i.RemoteItem.CreatedDateTime + } + return i.CreatedDateTime +} + +// GetLastModifiedDateTime returns a normalized LastModifiedDateTime of the item +func (i *Item) GetLastModifiedDateTime() Timestamp { + if i.IsRemote() && i.RemoteItem.LastModifiedDateTime != (Timestamp{}) { + return i.RemoteItem.LastModifiedDateTime + } + return i.LastModifiedDateTime +} + +// GetParentReferance returns a normalized ParentReferance of the item +func (i *Item) GetParentReferance() *ItemReference { + if i.IsRemote() && i.ParentReference == nil { + return i.RemoteItem.ParentReference + } + return i.ParentReference +} + +// IsRemote checks if item is a remote item +func (i *Item) IsRemote() bool { + return i.RemoteItem != nil +} diff --git a/.rclone_repo/backend/onedrive/onedrive.go b/.rclone_repo/backend/onedrive/onedrive.go new file mode 100755 index 0000000..9cfe81c --- /dev/null +++ b/.rclone_repo/backend/onedrive/onedrive.go @@ -0,0 +1,1509 @@ +// Package onedrive provides an interface to the Microsoft OneDrive +// object storage system. +package onedrive + +import ( + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/ncw/rclone/backend/onedrive/api" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/dircache" + "github.com/ncw/rclone/lib/oauthutil" + "github.com/ncw/rclone/lib/pacer" + "github.com/ncw/rclone/lib/readers" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" + "golang.org/x/oauth2" +) + +const ( + rclonePersonalClientID = "0000000044165769" + rclonePersonalEncryptedClientSecret = "ugVWLNhKkVT1-cbTRO-6z1MlzwdW6aMwpKgNaFG-qXjEn_WfDnG9TVyRA5yuoliU" + rcloneBusinessClientID = "52857fec-4bc2-483f-9f1b-5fe28e97532c" + rcloneBusinessEncryptedClientSecret = "6t4pC8l6L66SFYVIi8PgECDyjXy_ABo1nsTaE-Lr9LpzC6yT4vNOwHsakwwdEui0O6B0kX8_xbBLj91J" + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential + rootURLPersonal = "https://api.onedrive.com/v1.0/drive" // root URL for requests + discoveryServiceURL = "https://api.office.com/discovery/" + configResourceURL = "resource_url" +) + +// Globals +var ( + // Description of how to auth for this app for a personal account + oauthPersonalConfig = &oauth2.Config{ + Scopes: []string{ + "wl.signin", // Allow single sign-on capabilities + "wl.offline_access", // Allow receiving a refresh token + "onedrive.readwrite", // r/w perms to all of a user's OneDrive files + }, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://login.live.com/oauth20_authorize.srf", + TokenURL: "https://login.live.com/oauth20_token.srf", + }, + ClientID: rclonePersonalClientID, + ClientSecret: obscure.MustReveal(rclonePersonalEncryptedClientSecret), + RedirectURL: oauthutil.RedirectLocalhostURL, + } + + // Description of how to auth for this app for a business account + oauthBusinessConfig = &oauth2.Config{ + Endpoint: oauth2.Endpoint{ + AuthURL: "https://login.microsoftonline.com/common/oauth2/authorize", + TokenURL: "https://login.microsoftonline.com/common/oauth2/token", + }, + ClientID: rcloneBusinessClientID, + ClientSecret: obscure.MustReveal(rcloneBusinessEncryptedClientSecret), + RedirectURL: oauthutil.RedirectLocalhostURL, + } + oauthBusinessResource = oauth2.SetAuthURLParam("resource", discoveryServiceURL) + sharedURL = "https://api.onedrive.com/v1.0/drives" // root URL for remote shared resources +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "onedrive", + Description: "Microsoft OneDrive", + NewFs: NewFs, + Config: func(name string, m configmap.Mapper) { + // choose account type + fmt.Printf("Choose OneDrive account type?\n") + fmt.Printf(" * Say b for a OneDrive business account\n") + fmt.Printf(" * Say p for a personal OneDrive account\n") + isPersonal := config.Command([]string{"bBusiness", "pPersonal"}) == 'p' + + if isPersonal { + // for personal accounts we don't safe a field about the account + err := oauthutil.Config("onedrive", name, m, oauthPersonalConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + } else { + err := oauthutil.ConfigErrorCheck("onedrive", name, m, func(req *http.Request) oauthutil.AuthError { + var resp oauthutil.AuthError + + resp.Name = req.URL.Query().Get("error") + resp.Code = strings.Split(req.URL.Query().Get("error_description"), ":")[0] // error_description begins with XXXXXXXXXXXX: + resp.Description = strings.Join(strings.Split(req.URL.Query().Get("error_description"), ":")[1:], ":") + resp.HelpURL = "https://rclone.org/onedrive/#troubleshooting" + return resp + }, oauthBusinessConfig, oauthBusinessResource) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + return + } + + // Are we running headless? + if automatic, _ := m.Get(config.ConfigAutomatic); automatic != "" { + // Yes, okay we are done + return + } + + type serviceResource struct { + ServiceAPIVersion string `json:"serviceApiVersion"` + ServiceEndpointURI string `json:"serviceEndpointUri"` + ServiceResourceID string `json:"serviceResourceId"` + } + type serviceResponse struct { + Services []serviceResource `json:"value"` + } + + oAuthClient, _, err := oauthutil.NewClient(name, m, oauthBusinessConfig) + if err != nil { + log.Fatalf("Failed to configure OneDrive: %v", err) + return + } + srv := rest.NewClient(oAuthClient) + + opts := rest.Opts{ + Method: "GET", + RootURL: discoveryServiceURL, + Path: "/v2.0/me/services", + } + services := serviceResponse{} + resp, err := srv.CallJSON(&opts, nil, &services) + if err != nil { + fs.Errorf(nil, "Failed to query available services: %v", err) + return + } + if resp.StatusCode != 200 { + fs.Errorf(nil, "Failed to query available services: Got HTTP error code %d", resp.StatusCode) + return + } + + var resourcesURL []string + var resourcesID []string + + for _, service := range services.Services { + if service.ServiceAPIVersion == "v2.0" { + resourcesID = append(resourcesID, service.ServiceResourceID) + resourcesURL = append(resourcesURL, service.ServiceEndpointURI) + } + // we only support 2.0 API + fs.Infof(nil, "Skipping API %s endpoint %s", service.ServiceAPIVersion, service.ServiceEndpointURI) + } + + var foundService string + if len(resourcesID) == 0 { + fs.Errorf(nil, "No Service found") + return + } else if len(resourcesID) == 1 { + foundService = resourcesID[0] + } else { + foundService = config.Choose("Choose resource URL", resourcesID, resourcesURL, false) + } + + m.Set(configResourceURL, foundService) + oauthBusinessResource = oauth2.SetAuthURLParam("resource", foundService) + + // get the token from the inital config + // we need to update the token with a resource + // specific token we will query now + token, err := oauthutil.GetToken(name, m) + if err != nil { + fs.Errorf(nil, "Error while getting token: %s", err) + return + } + + // values for the token query + values := url.Values{} + values.Set("refresh_token", token.RefreshToken) + values.Set("grant_type", "refresh_token") + values.Set("resource", foundService) + values.Set("client_id", oauthBusinessConfig.ClientID) + values.Set("client_secret", oauthBusinessConfig.ClientSecret) + opts = rest.Opts{ + Method: "POST", + RootURL: oauthBusinessConfig.Endpoint.TokenURL, + ContentType: "application/x-www-form-urlencoded", + Body: strings.NewReader(values.Encode()), + } + + // tokenJSON is the struct representing the HTTP response from OAuth2 + // providers returning a token in JSON form. + // we are only interested in the new tokens, all other fields we don't care + type tokenJSON struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + } + jsonToken := tokenJSON{} + resp, err = srv.CallJSON(&opts, nil, &jsonToken) + if err != nil { + fs.Errorf(nil, "Failed to get resource token: %v", err) + return + } + if resp.StatusCode != 200 { + fs.Errorf(nil, "Failed to get resource token: Got HTTP error code %d", resp.StatusCode) + return + } + + // update the tokens + token.AccessToken = jsonToken.AccessToken + token.RefreshToken = jsonToken.RefreshToken + + // finally save them in the config + err = oauthutil.PutToken(name, m, token, true) + if err != nil { + fs.Errorf(nil, "Error while setting token: %s", err) + } + } + }, + Options: []fs.Option{{ + Name: config.ConfigClientID, + Help: "Microsoft App Client Id\nLeave blank normally.", + }, { + Name: config.ConfigClientSecret, + Help: "Microsoft App Client Secret\nLeave blank normally.", + }, { + Name: "chunk_size", + Help: "Chunk size to upload files with - must be multiple of 320k.", + Default: fs.SizeSuffix(10 * 1024 * 1024), + Advanced: true, + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + ChunkSize fs.SizeSuffix `config:"chunk_size"` + ResourceURL string `config:"resource_url"` +} + +// Fs represents a remote one drive +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + features *fs.Features // optional features + srv *rest.Client // the connection to the one drive server + dirCache *dircache.DirCache // Map of directory path to directory id + pacer *pacer.Pacer // pacer for API calls + tokenRenewer *oauthutil.Renew // renew the token on expiry + isBusiness bool // true if this is an OneDrive Business account +} + +// Object describes a one drive object +// +// Will definitely have info but maybe not meta +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + hasMetaData bool // whether info below has been set + size int64 // size of the object + modTime time.Time // modification time of the object + id string // ID of the object + sha1 string // SHA-1 of the object content + quickxorhash string // QuickXorHash of the object content + mimeType string // Content-Type of object from server (may not be as uploaded) +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("One drive root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// parsePath parses an one drive 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(resp *http.Response, err error) (bool, error) { + authRety := false + + if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { + authRety = true + fs.Debugf(nil, "Should retry: %v", err) + } + return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// readMetaDataForPath reads the metadata from the path +func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Response, err error) { + opts := rest.Opts{ + Method: "GET", + Path: "/root:/" + rest.URLPathEscape(replaceReservedChars(path)), + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &info) + return shouldRetry(resp, err) + }) + + return info, resp, err +} + +// errorHandler parses a non 2xx error response into an error +func errorHandler(resp *http.Response) error { + // Decode error response + errResponse := new(api.Error) + err := rest.DecodeJSON(resp, &errResponse) + if err != nil { + fs.Debugf(nil, "Couldn't decode error response: %v", err) + } + if errResponse.ErrorInfo.Code == "" { + errResponse.ErrorInfo.Code = resp.Status + } + return errResponse +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + if opt.ChunkSize%(320*1024) != 0 { + return nil, errors.Errorf("chunk size %d is not a multiple of 320k", opt.ChunkSize) + } + // if we have a resource URL it's a business account otherwise a personal one + isBusiness := opt.ResourceURL != "" + var rootURL string + var oauthConfig *oauth2.Config + if !isBusiness { + // personal account setup + oauthConfig = oauthPersonalConfig + rootURL = rootURLPersonal + } else { + // business account setup + oauthConfig = oauthBusinessConfig + rootURL = opt.ResourceURL + "_api/v2.0/drives/me" + sharedURL = opt.ResourceURL + "_api/v2.0/drives" + + // update the URL in the AuthOptions + oauthBusinessResource = oauth2.SetAuthURLParam("resource", opt.ResourceURL) + } + root = parsePath(root) + oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig) + if err != nil { + log.Fatalf("Failed to configure OneDrive: %v", err) + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + srv: rest.NewClient(oAuthClient).SetRoot(rootURL), + pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), + isBusiness: isBusiness, + } + f.features = (&fs.Features{ + CaseInsensitive: true, + // OneDrive for business doesn't support mime types properly + // so we disable it until resolved + // https://github.com/OneDrive/onedrive-api-docs/issues/643 + ReadMimeType: !f.isBusiness, + CanHaveEmptyDirectories: true, + }).Fill(f) + f.srv.SetErrorHandler(errorHandler) + + // Renew the token in the background + f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { + _, _, err := f.readMetaDataForPath("") + return err + }) + + // Get rootID + rootInfo, _, err := f.readMetaDataForPath("") + if err != nil || rootInfo.ID == "" { + return nil, errors.Wrap(err, "failed to get root") + } + + f.dirCache = dircache.New(root, rootInfo.ID, f) + + // Find the current root + err = f.dirCache.FindRoot(false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + newF := *f + newF.dirCache = dircache.New(newRoot, rootInfo.ID, &newF) + newF.root = newRoot + // Make new Fs which is the parent + err = newF.dirCache.FindRoot(false) + if err != nil { + // No root so return old f + return f, nil + } + _, err := newF.newObjectWithInfo(remote, nil) + if err != nil { + if err == fs.ErrorObjectNotFound { + // File doesn't exist so return old f + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return &newF, fs.ErrorIsFile + } + return f, nil +} + +// rootSlash returns root with a slash on if it is empty, otherwise empty string +func (f *Fs) rootSlash() string { + if f.root == "" { + return f.root + } + return f.root + "/" +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *api.Item) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + // Set info + err = o.setMetaData(info) + } else { + err = o.readMetaData() // reads info and meta, returning an error + } + if err != nil { + return nil, err + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) { + // fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf) + parent, ok := f.dirCache.GetInv(pathID) + if !ok { + return "", false, errors.New("couldn't find parent ID") + } + path := leaf + if parent != "" { + path = parent + "/" + path + } + if f.dirCache.FoundRoot() { + path = f.rootSlash() + path + } + info, resp, err := f.readMetaDataForPath(path) + if err != nil { + if resp != nil && resp.StatusCode == http.StatusNotFound { + return "", false, nil + } + return "", false, err + } + if info.GetFolder() == nil { + return "", false, errors.New("found file when looking for folder") + } + return info.GetID(), true, nil +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(dirID, leaf string) (newID string, err error) { + // fs.Debugf(f, "CreateDir(%q, %q)\n", dirID, leaf) + var resp *http.Response + var info *api.Item + opts := newOptsCall(dirID, "POST", "/children") + mkdir := api.CreateItemRequest{ + Name: replaceReservedChars(leaf), + ConflictBehavior: "fail", + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, &mkdir, &info) + return shouldRetry(resp, err) + }) + if err != nil { + //fmt.Printf("...Error %v\n", err) + return "", err + } + + //fmt.Printf("...Id %q\n", *info.Id) + return info.GetID(), nil +} + +// list the objects into the function supplied +// +// If directories is set it only sends directories +// User function to process a File item from listAll +// +// Should return true to finish processing +type listAllFn func(*api.Item) bool + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { + // Top parameter asks for bigger pages of data + // https://dev.onedrive.com/odata/optional-query-parameters.htm + opts := newOptsCall(dirID, "GET", "/children?top=1000") + +OUTER: + for { + var result api.ListChildrenResponse + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &result) + return shouldRetry(resp, err) + }) + if err != nil { + return found, errors.Wrap(err, "couldn't list files") + } + if len(result.Value) == 0 { + break + } + for i := range result.Value { + item := &result.Value[i] + isFolder := item.GetFolder() != nil + if isFolder { + if filesOnly { + continue + } + } else { + if directoriesOnly { + continue + } + } + if item.Deleted != nil { + continue + } + item.Name = restoreReservedChars(item.GetName()) + if fn(item) { + found = true + break OUTER + } + } + if result.NextLink == "" { + break + } + opts.Path = "" + opts.RootURL = result.NextLink + } + return +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + err = f.dirCache.FindRoot(false) + if err != nil { + return nil, err + } + directoryID, err := f.dirCache.FindDir(dir, false) + if err != nil { + return nil, err + } + var iErr error + _, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { + remote := path.Join(dir, info.GetName()) + folder := info.GetFolder() + if folder != nil { + // cache the directory ID for later lookups + id := info.GetID() + f.dirCache.Put(remote, id) + d := fs.NewDir(remote, time.Time(info.GetLastModifiedDateTime())).SetID(id) + if folder != nil { + d.SetItems(folder.ChildCount) + } + entries = append(entries, d) + } else { + o, err := f.newObjectWithInfo(remote, info) + if err != nil { + iErr = err + return true + } + entries = append(entries, o) + } + return false + }) + if err != nil { + return nil, err + } + if iErr != nil { + return nil, iErr + } + return entries, nil +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Returns the object, leaf, directoryID and error +// +// Used to create new objects +func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { + // Create the directory for the object if it doesn't exist + leaf, directoryID, err = f.dirCache.FindRootAndPath(remote, true) + if err != nil { + return nil, leaf, directoryID, err + } + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + } + return o, leaf, directoryID, nil +} + +// Put the object into the container +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime() + + o, _, _, err := f.createObject(remote, modTime, size) + if err != nil { + return nil, err + } + return o, o.Update(in, src, options...) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + err := f.dirCache.FindRoot(true) + if err != nil { + return err + } + if dir != "" { + _, err = f.dirCache.FindDir(dir, true) + } + return err +} + +// deleteObject removes an object by ID +func (f *Fs) deleteObject(id string) error { + opts := newOptsCall(id, "DELETE", "") + opts.NoResponse = true + + return f.pacer.Call(func() (bool, error) { + resp, err := f.srv.Call(&opts) + return shouldRetry(resp, err) + }) +} + +// purgeCheck removes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(dir string, check bool) error { + root := path.Join(f.root, dir) + if root == "" { + return errors.New("can't purge root directory") + } + dc := f.dirCache + err := dc.FindRoot(false) + if err != nil { + return err + } + rootID, err := dc.FindDir(dir, false) + if err != nil { + return err + } + if check { + // check to see if there are any items + found, err := f.listAll(rootID, false, false, func(item *api.Item) bool { + return true + }) + if err != nil { + return err + } + if found { + return fs.ErrorDirectoryNotEmpty + } + } + err = f.deleteObject(rootID) + if err != nil { + return err + } + f.dirCache.FlushDir(dir) + if err != nil { + return err + } + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + return f.purgeCheck(dir, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// waitForJob waits for the job with status in url to complete +func (f *Fs) waitForJob(location string, o *Object) error { + deadline := time.Now().Add(fs.Config.Timeout) + for time.Now().Before(deadline) { + opts := rest.Opts{ + Method: "GET", + RootURL: location, + IgnoreStatus: true, // Ignore the http status response since it seems to return valid info on 500 errors + } + var resp *http.Response + var err error + var body []byte + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.Call(&opts) + if err != nil { + return fserrors.ShouldRetry(err), err + } + body, err = rest.ReadBody(resp) + return fserrors.ShouldRetry(err), err + }) + if err != nil { + return err + } + // Try to decode the body first as an api.AsyncOperationStatus + var status api.AsyncOperationStatus + err = json.Unmarshal(body, &status) + if err != nil { + return errors.Wrapf(err, "async status result not JSON: %q", body) + } + // See if we decoded anything... + if !(status.Operation == "" && status.PercentageComplete == 0 && status.Status == "") { + if status.Status == "failed" || status.Status == "deleteFailed" { + return errors.Errorf("%s: async operation %q returned %q", o.remote, status.Operation, status.Status) + } + } else if resp.StatusCode == 200 { + var info api.Item + err = json.Unmarshal(body, &info) + if err != nil { + return errors.Wrapf(err, "async item result not JSON: %q", body) + } + return o.setMetaData(&info) + } + time.Sleep(1 * time.Second) + } + return errors.Errorf("async operation didn't complete after %v", fs.Config.Timeout) +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + err := srcObj.readMetaData() + if err != nil { + return nil, err + } + + srcPath := srcObj.fs.rootSlash() + srcObj.remote + dstPath := f.rootSlash() + remote + if strings.ToLower(srcPath) == strings.ToLower(dstPath) { + return nil, errors.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + // Copy the object + opts := newOptsCall(srcObj.id, "POST", "/action.copy") + opts.ExtraHeaders = map[string]string{"Prefer": "respond-async"} + opts.NoResponse = true + + id, _, _ := parseDirID(directoryID) + + replacedLeaf := replaceReservedChars(leaf) + copyReq := api.CopyItemRequest{ + Name: &replacedLeaf, + ParentReference: api.ItemReference{ + ID: id, + }, + } + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, ©Req, nil) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + + // read location header + location := resp.Header.Get("Location") + if location == "" { + return nil, errors.New("didn't receive location header in copy response") + } + + // Wait for job to finish + err = f.waitForJob(location, dstObj) + if err != nil { + return nil, err + } + + // Copy does NOT copy the modTime from the source and there seems to + // be no way to set date before + // This will create TWO versions on OneDrive + err = dstObj.SetModTime(srcObj.ModTime()) + if err != nil { + return nil, err + } + + return dstObj, nil +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + return f.purgeCheck("", false) +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + // Move the object + opts := newOptsCall(srcObj.id, "PATCH", "") + + id, _, _ := parseDirID(directoryID) + + move := api.MoveItemRequest{ + Name: replaceReservedChars(leaf), + ParentReference: &api.ItemReference{ + ID: id, + }, + // We set the mod time too as it gets reset otherwise + FileSystemInfo: &api.FileSystemInfoFacet{ + CreatedDateTime: api.Timestamp(srcObj.modTime), + LastModifiedDateTime: api.Timestamp(srcObj.modTime), + }, + } + var resp *http.Response + var info api.Item + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, &move, &info) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + + err = dstObj.setMetaData(&info) + if err != nil { + return nil, err + } + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := path.Join(srcFs.root, srcRemote) + dstPath := path.Join(f.root, dstRemote) + + // Refuse to move to or from the root + if srcPath == "" || dstPath == "" { + fs.Debugf(src, "DirMove error: Can't move root") + return errors.New("can't move root directory") + } + + // find the root src directory + err := srcFs.dirCache.FindRoot(false) + if err != nil { + return err + } + + // find the root dst directory + if dstRemote != "" { + err = f.dirCache.FindRoot(true) + if err != nil { + return err + } + } else { + if f.dirCache.FoundRoot() { + return fs.ErrorDirExists + } + } + + // Find ID of dst parent, creating subdirs if necessary + var leaf, dstDirectoryID string + findPath := dstRemote + if dstRemote == "" { + findPath = f.root + } + leaf, dstDirectoryID, err = f.dirCache.FindPath(findPath, true) + if err != nil { + return err + } + parsedDstDirID, _, _ := parseDirID(dstDirectoryID) + + // Check destination does not exist + if dstRemote != "" { + _, err = f.dirCache.FindDir(dstRemote, false) + if err == fs.ErrorDirNotFound { + // OK + } else if err != nil { + return err + } else { + return fs.ErrorDirExists + } + } + + // Find ID of src + srcID, err := srcFs.dirCache.FindDir(srcRemote, false) + if err != nil { + return err + } + + // Get timestamps of src so they can be preserved + srcInfo, _, err := srcFs.readMetaDataForPath(srcPath) + if err != nil { + return err + } + + // Do the move + opts := newOptsCall(srcID, "PATCH", "") + move := api.MoveItemRequest{ + Name: replaceReservedChars(leaf), + ParentReference: &api.ItemReference{ + ID: parsedDstDirID, + }, + // We set the mod time too as it gets reset otherwise + FileSystemInfo: &api.FileSystemInfoFacet{ + CreatedDateTime: srcInfo.CreatedDateTime, + LastModifiedDateTime: srcInfo.LastModifiedDateTime, + }, + } + var resp *http.Response + var info api.Item + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, &move, &info) + return shouldRetry(resp, err) + }) + if err != nil { + return err + } + + srcFs.dirCache.FlushDir(srcRemote) + return nil +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *Fs) DirCacheFlush() { + f.dirCache.ResetRoot() +} + +// About gets quota information +func (f *Fs) About() (usage *fs.Usage, err error) { + var drive api.Drive + opts := rest.Opts{ + Method: "GET", + Path: "", + } + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &drive) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "about failed") + } + q := drive.Quota + usage = &fs.Usage{ + Total: fs.NewUsageValue(q.Total), // quota of bytes that can be used + Used: fs.NewUsageValue(q.Used), // bytes in use + Trashed: fs.NewUsageValue(q.Deleted), // bytes in trash + Free: fs.NewUsageValue(q.Remaining), // bytes which can be uploaded before reaching the quota + } + return usage, nil +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + if f.isBusiness { + return hash.Set(hash.QuickXorHash) + } + return hash.Set(hash.SHA1) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// srvPath returns a path for use in server +func (o *Object) srvPath() string { + return replaceReservedChars(o.fs.rootSlash() + o.remote) +} + +// Hash returns the SHA-1 of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if o.fs.isBusiness { + if t != hash.QuickXorHash { + return "", hash.ErrUnsupported + } + return o.quickxorhash, nil + } + if t != hash.SHA1 { + return "", hash.ErrUnsupported + } + return o.sha1, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return 0 + } + return o.size +} + +// setMetaData sets the metadata from info +func (o *Object) setMetaData(info *api.Item) (err error) { + if info.GetFolder() != nil { + return errors.Wrapf(fs.ErrorNotAFile, "%q", o.remote) + } + o.hasMetaData = true + o.size = info.GetSize() + + // Docs: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/hashes + // + // We use SHA1 for onedrive personal and QuickXorHash for onedrive for business + file := info.GetFile() + if file != nil { + o.mimeType = file.MimeType + if file.Hashes.Sha1Hash != "" { + o.sha1 = strings.ToLower(file.Hashes.Sha1Hash) + } + if file.Hashes.QuickXorHash != "" { + h, err := base64.StdEncoding.DecodeString(file.Hashes.QuickXorHash) + if err != nil { + fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", file.Hashes.QuickXorHash, err) + } else { + o.quickxorhash = hex.EncodeToString(h) + } + } + } + fileSystemInfo := info.GetFileSystemInfo() + if fileSystemInfo != nil { + o.modTime = time.Time(fileSystemInfo.LastModifiedDateTime) + } else { + o.modTime = time.Time(info.GetLastModifiedDateTime()) + } + o.id = info.GetID() + return nil +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData() (err error) { + if o.hasMetaData { + return nil + } + info, _, err := o.fs.readMetaDataForPath(o.srvPath()) + if err != nil { + if apiErr, ok := err.(*api.Error); ok { + if apiErr.ErrorInfo.Code == "itemNotFound" { + return fs.ErrorObjectNotFound + } + } + return err + } + return o.setMetaData(info) +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// setModTime sets the modification time of the local fs object +func (o *Object) setModTime(modTime time.Time) (*api.Item, error) { + var opts rest.Opts + _, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false) + _, drive, rootURL := parseDirID(directoryID) + if drive != "" { + opts = rest.Opts{ + Method: "PATCH", + RootURL: rootURL, + Path: "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()), + } + } else { + opts = rest.Opts{ + Method: "PATCH", + Path: "/root:/" + rest.URLPathEscape(o.srvPath()), + } + } + update := api.SetFileSystemInfo{ + FileSystemInfo: api.FileSystemInfoFacet{ + CreatedDateTime: api.Timestamp(modTime), + LastModifiedDateTime: api.Timestamp(modTime), + }, + } + var info *api.Item + err := o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.CallJSON(&opts, &update, &info) + return shouldRetry(resp, err) + }) + return info, err +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + info, err := o.setModTime(modTime) + if err != nil { + return err + } + return o.setMetaData(info) +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + if o.id == "" { + return nil, errors.New("can't download - no id") + } + fs.FixRangeOption(options, o.size) + var resp *http.Response + opts := newOptsCall(o.id, "GET", "/content") + opts.Options = options + + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusOK && resp.ContentLength > 0 && resp.Header.Get("Content-Range") == "" { + //Overwrite size with actual size since size readings from Onedrive is unreliable. + o.size = resp.ContentLength + } + return resp.Body, err +} + +// createUploadSession creates an upload session for the object +func (o *Object) createUploadSession(modTime time.Time) (response *api.CreateUploadResponse, err error) { + leaf, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false) + id, drive, rootURL := parseDirID(directoryID) + var opts rest.Opts + if drive != "" { + opts = rest.Opts{ + Method: "POST", + RootURL: rootURL, + Path: "/" + drive + "/items/" + id + ":/" + rest.URLPathEscape(leaf) + ":/upload.createSession", + } + } else { + opts = rest.Opts{ + Method: "POST", + Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/upload.createSession", + } + } + createRequest := api.CreateUploadRequest{} + createRequest.Item.FileSystemInfo.CreatedDateTime = api.Timestamp(modTime) + createRequest.Item.FileSystemInfo.LastModifiedDateTime = api.Timestamp(modTime) + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(&opts, &createRequest, &response) + return shouldRetry(resp, err) + }) + return response, err +} + +// uploadFragment uploads a part +func (o *Object) uploadFragment(url string, start int64, totalSize int64, chunk io.ReadSeeker, chunkSize int64) (info *api.Item, err error) { + opts := rest.Opts{ + Method: "PUT", + RootURL: url, + ContentLength: &chunkSize, + ContentRange: fmt.Sprintf("bytes %d-%d/%d", start, start+chunkSize-1, totalSize), + Body: chunk, + } + // var response api.UploadFragmentResponse + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + _, _ = chunk.Seek(0, io.SeekStart) + resp, err = o.fs.srv.Call(&opts) + if resp != nil { + defer fs.CheckClose(resp.Body, &err) + } + retry, err := shouldRetry(resp, err) + if !retry && resp != nil { + if resp.StatusCode == 200 || resp.StatusCode == 201 { + // we are done :) + // read the item + info = &api.Item{} + return false, json.NewDecoder(resp.Body).Decode(info) + } + } + return retry, err + }) + return info, err +} + +// cancelUploadSession cancels an upload session +func (o *Object) cancelUploadSession(url string) (err error) { + opts := rest.Opts{ + Method: "DELETE", + RootURL: url, + NoResponse: true, + } + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return shouldRetry(resp, err) + }) + return +} + +// uploadMultipart uploads a file using multipart upload +func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) { + // Create upload session + fs.Debugf(o, "Starting multipart upload") + session, err := o.createUploadSession(modTime) + if err != nil { + return nil, err + } + uploadURL := session.UploadURL + + // Cancel the session if something went wrong + defer func() { + if err != nil { + fs.Debugf(o, "Cancelling multipart upload: %v", err) + cancelErr := o.cancelUploadSession(uploadURL) + if cancelErr != nil { + fs.Logf(o, "Failed to cancel multipart upload: %v", err) + } + } + }() + + // Upload the chunks + remaining := size + position := int64(0) + for remaining > 0 { + n := int64(o.fs.opt.ChunkSize) + if remaining < n { + n = remaining + } + seg := readers.NewRepeatableReader(io.LimitReader(in, n)) + fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n) + info, err = o.uploadFragment(uploadURL, position, size, seg, n) + if err != nil { + return nil, err + } + remaining -= n + position += n + } + + return info, nil +} + +// uploadSinglepart uploads a file as a single part +func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) { + var resp *http.Response + var opts rest.Opts + _, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false) + _, drive, rootURL := parseDirID(directoryID) + if drive != "" { + opts = rest.Opts{ + Method: "PUT", + RootURL: rootURL, + Path: "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content", + ContentLength: &size, + Body: in, + } + } else { + opts = rest.Opts{ + Method: "PUT", + Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content", + ContentLength: &size, + Body: in, + } + } + // for go1.8 (see release notes) we must nil the Body if we want a + // "Content-Length: 0" header which onedrive requires for all files. + if size == 0 { + opts.Body = nil + } + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(&opts, nil, &info) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + + err = o.setMetaData(info) + if err != nil { + return nil, err + } + // Set the mod time now and read metadata + return o.setModTime(modTime) +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + o.fs.tokenRenewer.Start() + defer o.fs.tokenRenewer.Stop() + + size := src.Size() + modTime := src.ModTime() + + var info *api.Item + if size <= 0 { + // This is for 0 length files, or files with an unknown size + info, err = o.uploadSinglepart(in, size, modTime) + } else { + info, err = o.uploadMultipart(in, size, modTime) + } + if err != nil { + return err + } + return o.setMetaData(info) +} + +// Remove an object +func (o *Object) Remove() error { + return o.fs.deleteObject(o.id) +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + return o.mimeType +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.id +} + +func newOptsCall(id string, method string, route string) (opts rest.Opts) { + id, drive, rootURL := parseDirID(id) + + if drive != "" { + return rest.Opts{ + Method: method, + RootURL: rootURL, + Path: "/" + drive + "/items/" + id + route, + } + } + return rest.Opts{ + Method: method, + Path: "/items/" + id + route, + } +} + +func parseDirID(ID string) (string, string, string) { + if strings.Index(ID, "#") >= 0 { + s := strings.Split(ID, "#") + return s[1], s[0], sharedURL + } + return ID, "", "" +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = &Object{} + _ fs.IDer = &Object{} +) diff --git a/.rclone_repo/backend/onedrive/onedrive_test.go b/.rclone_repo/backend/onedrive/onedrive_test.go new file mode 100755 index 0000000..07eccc0 --- /dev/null +++ b/.rclone_repo/backend/onedrive/onedrive_test.go @@ -0,0 +1,17 @@ +// Test OneDrive filesystem interface +package onedrive_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/onedrive" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestOneDrive:", + NilObject: (*onedrive.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/onedrive/quickxorhash/quickxorhash.go b/.rclone_repo/backend/onedrive/quickxorhash/quickxorhash.go new file mode 100755 index 0000000..b7976b7 --- /dev/null +++ b/.rclone_repo/backend/onedrive/quickxorhash/quickxorhash.go @@ -0,0 +1,202 @@ +// Package quickxorhash provides the quickXorHash algorithm which is a +// quick, simple non-cryptographic hash algorithm that works by XORing +// the bytes in a circular-shifting fashion. +// +// It is used by Microsoft Onedrive for Business to hash data. +// +// See: https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash +package quickxorhash + +// This code was ported from the code snippet linked from +// https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash +// Which has the copyright + +// ------------------------------------------------------------------------------ +// Copyright (c) 2016 Microsoft Corporation +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// ------------------------------------------------------------------------------ + +import ( + "hash" +) + +const ( + // BlockSize is the preferred size for hashing + BlockSize = 64 + // Size of the output checksum + Size = 20 + bitsInLastCell = 32 + shift = 11 + widthInBits = 8 * Size + dataSize = (widthInBits-1)/64 + 1 +) + +type quickXorHash struct { + data [dataSize]uint64 + lengthSoFar uint64 + shiftSoFar int +} + +// New returns a new hash.Hash computing the quickXorHash checksum. +func New() hash.Hash { + return &quickXorHash{} +} + +// Write (via the embedded io.Writer interface) adds more data to the running hash. +// It never returns an error. +// +// Write writes len(p) bytes from p to the underlying data stream. It returns +// the number of bytes written from p (0 <= n <= len(p)) and any error +// encountered that caused the write to stop early. Write must return a non-nil +// error if it returns n < len(p). Write must not modify the slice data, even +// temporarily. +// +// Implementations must not retain p. +func (q *quickXorHash) Write(p []byte) (n int, err error) { + currentshift := q.shiftSoFar + + // The bitvector where we'll start xoring + vectorArrayIndex := currentshift / 64 + + // The position within the bit vector at which we begin xoring + vectorOffset := currentshift % 64 + iterations := len(p) + if iterations > widthInBits { + iterations = widthInBits + } + + for i := 0; i < iterations; i++ { + isLastCell := vectorArrayIndex == len(q.data)-1 + var bitsInVectorCell int + if isLastCell { + bitsInVectorCell = bitsInLastCell + } else { + bitsInVectorCell = 64 + } + + // There's at least 2 bitvectors before we reach the end of the array + if vectorOffset <= bitsInVectorCell-8 { + for j := i; j < len(p); j += widthInBits { + q.data[vectorArrayIndex] ^= uint64(p[j]) << uint(vectorOffset) + } + } else { + index1 := vectorArrayIndex + var index2 int + if isLastCell { + index2 = 0 + } else { + index2 = vectorArrayIndex + 1 + } + low := byte(bitsInVectorCell - vectorOffset) + + xoredByte := byte(0) + for j := i; j < len(p); j += widthInBits { + xoredByte ^= p[j] + } + q.data[index1] ^= uint64(xoredByte) << uint(vectorOffset) + q.data[index2] ^= uint64(xoredByte) >> low + } + vectorOffset += shift + for vectorOffset >= bitsInVectorCell { + if isLastCell { + vectorArrayIndex = 0 + } else { + vectorArrayIndex = vectorArrayIndex + 1 + } + vectorOffset -= bitsInVectorCell + } + } + + // Update the starting position in a circular shift pattern + q.shiftSoFar = (q.shiftSoFar + shift*(len(p)%widthInBits)) % widthInBits + + q.lengthSoFar += uint64(len(p)) + + return len(p), nil +} + +// Calculate the current checksum +func (q *quickXorHash) checkSum() (h [Size]byte) { + // Output the data as little endian bytes + ph := 0 + for _, d := range q.data[:len(q.data)-1] { + _ = h[ph+7] // bounds check + h[ph+0] = byte(d >> (8 * 0)) + h[ph+1] = byte(d >> (8 * 1)) + h[ph+2] = byte(d >> (8 * 2)) + h[ph+3] = byte(d >> (8 * 3)) + h[ph+4] = byte(d >> (8 * 4)) + h[ph+5] = byte(d >> (8 * 5)) + h[ph+6] = byte(d >> (8 * 6)) + h[ph+7] = byte(d >> (8 * 7)) + ph += 8 + } + // remaining 32 bits + d := q.data[len(q.data)-1] + h[Size-4] = byte(d >> (8 * 0)) + h[Size-3] = byte(d >> (8 * 1)) + h[Size-2] = byte(d >> (8 * 2)) + h[Size-1] = byte(d >> (8 * 3)) + + // XOR the file length with the least significant bits in little endian format + d = q.lengthSoFar + h[Size-8] ^= byte(d >> (8 * 0)) + h[Size-7] ^= byte(d >> (8 * 1)) + h[Size-6] ^= byte(d >> (8 * 2)) + h[Size-5] ^= byte(d >> (8 * 3)) + h[Size-4] ^= byte(d >> (8 * 4)) + h[Size-3] ^= byte(d >> (8 * 5)) + h[Size-2] ^= byte(d >> (8 * 6)) + h[Size-1] ^= byte(d >> (8 * 7)) + + return h +} + +// Sum appends the current hash to b and returns the resulting slice. +// It does not change the underlying hash state. +func (q *quickXorHash) Sum(b []byte) []byte { + hash := q.checkSum() + return append(b, hash[:]...) +} + +// Reset resets the Hash to its initial state. +func (q *quickXorHash) Reset() { + *q = quickXorHash{} +} + +// Size returns the number of bytes Sum will return. +func (q *quickXorHash) Size() int { + return Size +} + +// BlockSize returns the hash's underlying block size. +// The Write method must be able to accept any amount +// of data, but it may operate more efficiently if all writes +// are a multiple of the block size. +func (q *quickXorHash) BlockSize() int { + return BlockSize +} + +// Sum returns the quickXorHash checksum of the data. +func Sum(data []byte) [Size]byte { + var d quickXorHash + _, _ = d.Write(data) + return d.checkSum() +} diff --git a/.rclone_repo/backend/onedrive/quickxorhash/quickxorhash_test.go b/.rclone_repo/backend/onedrive/quickxorhash/quickxorhash_test.go new file mode 100755 index 0000000..51f6972 --- /dev/null +++ b/.rclone_repo/backend/onedrive/quickxorhash/quickxorhash_test.go @@ -0,0 +1,168 @@ +package quickxorhash + +import ( + "encoding/base64" + "fmt" + "hash" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var testVectors = []struct { + size int + in string + out string +}{ + {0, ``, "AAAAAAAAAAAAAAAAAAAAAAAAAAA="}, + {1, `Sg==`, "SgAAAAAAAAAAAAAAAQAAAAAAAAA="}, + {2, `tbQ=`, "taAFAAAAAAAAAAAAAgAAAAAAAAA="}, + {3, `0pZP`, "0rDEEwAAAAAAAAAAAwAAAAAAAAA="}, + {4, `jRRDVA==`, "jaDAEKgAAAAAAAAABAAAAAAAAAA="}, + {5, `eAV52qE=`, "eChAHrQRCgAAAAAABQAAAAAAAAA="}, + {6, `luBZlaT6`, "lgBHFipBCn0AAAAABgAAAAAAAAA="}, + {7, `qaApEj66lw==`, "qQBFCiTgA11cAgAABwAAAAAAAAA="}, + {8, `/aNzzCFPS/A=`, "/RjFHJgRgicsAR4ACAAAAAAAAAA="}, + {9, `n6Neh7p6fFgm`, "nxiFFw6hCz3wAQsmCQAAAAAAAAA="}, + {10, `J9iPGCbfZSTNyw==`, "J8DGIzBggm+UgQTNUgYAAAAAAAA="}, + {11, `i+UZyUGJKh+ISbk=`, "iyhHBpIRhESo4AOIQ0IuAAAAAAA="}, + {12, `h490d57Pqz5q2rtT`, "h3gEHe7giWeswgdq3MYupgAAAAA="}, + {13, `vPgoDjOfO6fm71RxLw==`, "vMAHChwwg0/s4BTmdQcV4vACAAA="}, + {14, `XoJ1AsoR4fDYJrDqYs4=`, "XhBEHQSgjAiEAx7YPgEs1CEGZwA="}, + {15, `gQaybEqS/4UlDc8e4IJm`, "gDCALNigBEn8oxAlZ8AzPAAOQZg="}, + {16, `2fuxhBJXtpWFe8dOfdGeHw==`, "O9tHLAghgSvYohKFyMMxnNCHaHg="}, + {17, `XBV6YKU9V7yMakZnFIxIkuU=`, "HbplHsBQih5cgReMQYMRzkABRiA="}, + {18, `XJZSOiNO2bmfKnTKD7fztcQX`, "/6ZArHQwAidkIxefQgEdlPGAW8w="}, + {19, `g8VtAh+2Kf4k0kY5tzji2i2zmA==`, "wDNrgwHWAVukwB8kg4YRcnALHIg="}, + {20, `T6LYJIfDh81JrAK309H2JMJTXis=`, "zBTHrspn3mEcohlJdIUAbjGNaNg="}, + {21, `DWAAX5/CIfrmErgZa8ot6ZraeSbu`, "LR2Z0PjuRYGKQB/mhQAuMrAGZbQ="}, + {22, `N9abi3qy/mC1THZuVLHPpx7SgwtLOA==`, "1KTYttCBEen8Hwy1doId3ECFWDw="}, + {23, `LlUe7wHerLqEtbSZLZgZa9u0m7hbiFs=`, "TqVZpxs3cN61BnuFvwUtMtECTGQ="}, + {24, `bU2j/0XYdgfPFD4691jV0AOUEUPR4Z5E`, "bnLBiLpVgnxVkXhNsIAPdHAPLFQ="}, + {25, `lScPwPsyUsH2T1Qsr31wXtP55Wqbe47Uyg==`, "VDMSy8eI26nBHCB0e8gVWPCKPsA="}, + {26, `rJaKh1dLR1k+4hynliTZMGf8Nd4qKKoZiAM=`, "r7bjwkl8OYQeNaMcCY8fTmEJEmQ="}, + {27, `pPsT0CPmHrd3Frsnva1pB/z1ytARLeHEYRCo`, "Rdg7rCcDomL59pL0s6GuTvqLVqQ="}, + {28, `wSRChaqmrsnMrfB2yqI43eRWbro+f9kBvh+01w==`, "YTtloIi6frI7HX3vdLvE7I2iUOA="}, + {29, `apL67KMIRxQeE9k1/RuW09ppPjbF1WeQpTjSWtI=`, "CIpedls+ZlSQ654fl+X26+Q7LVU="}, + {30, `53yx0/QgMTVb7OOzHRHbkS7ghyRc+sIXxi7XHKgT`, "zfJtLGFgR9DB3Q64fAFIp+S5iOY="}, + {31, `PwXNnutoLLmxD8TTog52k8cQkukmT87TTnDipKLHQw==`, "PTaGs7yV3FUyBy/SfU6xJRlCJlI="}, + {32, `NbYXsp5/K6mR+NmHwExjvWeWDJFnXTKWVlzYHoesp2E=`, "wjuAuWDiq04qDt1R8hHWDDcwVoQ="}, + {33, `qQ70RB++JAR5ljNv3lJt1PpqETPsckopfonItu18Cr3E`, "FkJaeg/0Z5+euShYlLpE2tJh+Lo="}, + {34, `RhzSatQTQ9/RFvpHyQa1WLdkr3nIk6MjJUma998YRtp44A==`, "SPN2D29reImAqJezlqV2DLbi8tk="}, + {35, `DND1u1uZ5SqZVpRUk6NxSUdVo7IjjL9zs4A1evDNCDLcXWc=`, "S6lBk2hxI2SWBfn7nbEl7D19UUs="}, + {36, `jEi62utFz69JMYHjg1iXy7oO6ZpZSLcVd2B+pjm6BGsv/CWi`, "s0lYU9tr/bp9xsnrrjYgRS5EvV8="}, + {37, `hfS3DZZnhy0hv7nJdXLv/oJOtIgAuP9SInt/v8KeuO4/IvVh4A==`, "CV+HQCdd2A/e/vdi12f2UU55GLA="}, + {38, `EkPQAC6ymuRrYjIXD/LT/4Vb+7aTjYVZOHzC8GPCEtYDP0+T3Nc=`, "kE9H9sEmr3vHBYUiPbvsrcDgSEo="}, + {39, `vtBOGIENG7yQ/N7xNWPNIgy66Gk/I2Ur/ZhdFNUK9/1FCZuu/KeS`, "+Fgp3HBimtCzUAyiinj3pkarYTk="}, + {40, `YnF4smoy9hox2jBlJ3VUa4qyCRhOZbWcmFGIiszTT4zAdYHsqJazyg==`, "arkIn+ELddmE8N34J9ydyFKW+9w="}, + {41, `0n7nl3YJtipy6yeUbVPWtc2h45WbF9u8hTz5tNwj3dZZwfXWkk+GN3g=`, "YJLNK7JR64j9aODWfqDvEe/u6NU="}, + {42, `FnIIPHayc1pHkY4Lh8+zhWwG8xk6Knk/D3cZU1/fOUmRAoJ6CeztvMOL`, "22RPOylMtdk7xO/QEQiMli4ql0k="}, + {43, `J82VT7ND0Eg1MorSfJMUhn+qocF7PsUpdQAMrDiHJ2JcPZAHZ2nyuwjoKg==`, "pOR5eYfwCLRJbJsidpc1rIJYwtM="}, + {44, `Zbu+78+e35ZIymV5KTDdub5McyI3FEO8fDxs62uWHQ9U3Oh3ZqgaZ30SnmQ=`, "DbvbTkgNTgWRqRidA9r1jhtUjro="}, + {45, `lgybK3Da7LEeY5aeeNrqcdHvv6mD1W4cuQ3/rUj2C/CNcSI0cAMw6vtpVY3y`, "700RQByn1lRQSSme9npQB/Ye+bY="}, + {46, `jStZgKHv4QyJLvF2bYbIUZi/FscHALfKHAssTXkrV1byVR9eACwW9DNZQRHQwg==`, "uwN55He8xgE4g93dH9163xPew4U="}, + {47, `V1PSud3giF5WW72JB/bgtltsWtEB5V+a+wUALOJOGuqztzVXUZYrvoP3XV++gM0=`, "U+3ZfUF/6mwOoHJcSHkQkckfTDA="}, + {48, `VXs4t4tfXGiWAL6dlhEMm0YQF0f2w9rzX0CvIVeuW56o6/ec2auMpKeU2VeteEK5`, "sq24lSf7wXLH8eigHl07X+qPTps="}, + {49, `bLUn3jLH+HFUsG3ptWTHgNvtr3eEv9lfKBf0jm6uhpqhRwtbEQ7Ovj/hYQf42zfdtQ==`, "uC8xrnopGiHebGuwgq607WRQyxQ="}, + {50, `4SVmjtXIL8BB8SfkbR5Cpaljm2jpyUfAhIBf65XmKxHlz9dy5XixgiE/q1lv+esZW/E=`, "wxZ0rxkMQEnRNAp8ZgEZLT4RdLM="}, + {51, `pMljctlXeFUqbG3BppyiNbojQO3ygg6nZPeUZaQcVyJ+Clgiw3Q8ntLe8+02ZSfyCc39`, "aZEPmNvOXnTt7z7wt+ewV7QGMlg="}, + {52, `C16uQlxsHxMWnV2gJhFPuJ2/guZ4N1YgmNvAwL1yrouGQtwieGx8WvZsmYRnX72JnbVtTw==`, "QtlSNqXhVij64MMhKJ3EsDFB/z8="}, + {53, `7ZVDOywvrl3L0GyKjjcNg2CcTI81n2CeUbzdYWcZOSCEnA/xrNHpiK01HOcGh3BbxuS4S6g=`, "4NznNJc4nmXeApfiCFTq/H5LbHw="}, + {54, `JXm2tTVqpYuuz2Cc+ZnPusUb8vccPGrzWK2oVwLLl/FjpFoxO9FxGlhnB08iu8Q/XQSdzHn+`, "IwE5+2pKNcK366I2k2BzZYPibSI="}, + {55, `TiiU1mxzYBSGZuE+TX0l9USWBilQ7dEml5lLrzNPh75xmhjIK8SGqVAkvIMgAmcMB+raXdMPZg==`, "yECGHtgR128ScP4XlvF96eLbIBE="}, + {56, `zz+Q4zi6wh0fCJUFU9yUOqEVxlIA93gybXHOtXIPwQQ44pW4fyh6BRgc1bOneRuSWp85hwlTJl8=`, "+3Ef4D6yuoC8J+rbFqU1cegverE="}, + {57, `sa6SHK9z/G505bysK5KgRO2z2cTksDkLoFc7sv0tWBmf2G2mCiozf2Ce6EIO+W1fRsrrtn/eeOAV`, "xZg1CwMNAjN0AIXw2yh4+1N3oos="}, + {58, `0qx0xdyTHhnKJ22IeTlAjRpWw6y2sOOWFP75XJ7cleGJQiV2kyrmQOST4DGHIL0qqA7sMOdzKyTV +iw==`, "bS0tRYPkP1Gfc+ZsBm9PMzPunG8="}, + {59, `QuzaF0+5ooig6OLEWeibZUENl8EaiXAQvK9UjBEauMeuFFDCtNcGs25BDtJGGbX90gH4VZvCCDNC +q4s=`, "rggokuJq1OGNOfB6aDp2g4rdPgw="}, + {60, `+wg2x23GZQmMLkdv9MeAdettIWDmyK6Wr+ba23XD+Pvvq1lIMn9QIQT4Z7QHJE3iC/ZMFgaId9VA +yY3d`, "ahQbTmOdiKUNdhYRHgv5/Ky+Y6k="}, + {61, `y0ydRgreRQwP95vpNP92ioI+7wFiyldHRbr1SfoPNdbKGFA0lBREaBEGNhf9yixmfE+Azo2AuROx +b7Yc7g==`, "cJKFc0dXfiN4hMg1lcMf5E4gqvo="}, + {62, `LxlVvGXSQlSubK8r0pGf9zf7s/3RHe75a2WlSXQf3gZFR/BtRnR7fCIcaG//CbGfodBFp06DBx/S +9hUV8Bk=`, "NwuwhhRWX8QZ/vhWKWgQ1+rNomI="}, + {63, `L+LSB8kmGMnHaWVA5P/+qFnfQliXvgJW7d2JGAgT6+koi5NQujFW1bwQVoXrBVyob/gBxGizUoJM +gid5gGNo`, "ndX/KZBtFoeO3xKeo1ajO/Jy+rY="}, + {64, `Mb7EGva2rEE5fENDL85P+BsapHEEjv2/siVhKjvAQe02feExVOQSkfmuYzU/kTF1MaKjPmKF/w+c +bvwfdWL8aQ==`, "n1anP5NfvD4XDYWIeRPW3ZkPv1Y="}, + {111, `jyibxJSzO6ZiZ0O1qe3tG/bvIAYssvukh9suIT5wEy1JBINVgPiqdsTW0cOpP0aUfP7mgqLfADkz +I/m/GgCuVhr8oFLrOCoTx1/psBOWwhltCbhUx51Icm9aH8tY4Z3ccU+6BKpYQkLCy0B/A9Zc`, "hZfLIilSITC6N3e3tQ/iSgEzkto="}, + {128, `ikwCorI7PKWz17EI50jZCGbV9JU2E8bXVfxNMg5zdmqSZ2NlsQPp0kqYIPjzwTg1MBtfWPg53k0h +0P2naJNEVgrqpoHTfV2b3pJ4m0zYPTJmUX4Bg/lOxcnCxAYKU29Y5F0U8Quz7ZXFBEweftXxJ7RS +4r6N7BzJrPsLhY7hgck=`, "imAoFvCWlDn4yVw3/oq1PDbbm6U="}, + {222, `PfxMcUd0vIW6VbHG/uj/Y0W6qEoKmyBD0nYebEKazKaKG+UaDqBEcmQjbfQeVnVLuodMoPp7P7TR +1htX5n2VnkHh22xDyoJ8C/ZQKiSNqQfXvh83judf4RVr9exJCud8Uvgip6aVZTaPrJHVjQhMCp/d +EnGvqg0oN5OVkM2qqAXvA0teKUDhgNM71sDBVBCGXxNOR2bpbD1iM4dnuT0ey4L+loXEHTL0fqMe +UcEi2asgImnlNakwenDzz0x57aBwyq3AspCFGB1ncX4yYCr/OaCcS5OKi/00WH+wNQU3`, "QX/YEpG0gDsmhEpCdWhsxDzsfVE="}, + {256, `qwGf2ESubE5jOUHHyc94ORczFYYbc2OmEzo+hBIyzJiNwAzC8PvJqtTzwkWkSslgHFGWQZR2BV5+ +uYTrYT7HVwRM40vqfj0dBgeDENyTenIOL1LHkjtDKoXEnQ0mXAHoJ8PjbNC93zi5TovVRXTNzfGE +s5dpWVqxUzb5lc7dwkyvOluBw482mQ4xrzYyIY1t+//OrNi1ObGXuUw2jBQOFfJVj2Y6BOyYmfB1 +y36eBxi3zxeG5d5NYjm2GSh6e08QMAwu3zrINcqIzLOuNIiGXBtl7DjKt7b5wqi4oFiRpZsCyx2s +mhSrdrtK/CkdU6nDN+34vSR/M8rZpWQdBE7a8g==`, "WYT9JY3JIo/pEBp+tIM6Gt2nyTM="}, + {333, `w0LGhqU1WXFbdavqDE4kAjEzWLGGzmTNikzqnsiXHx2KRReKVTxkv27u3UcEz9+lbMvYl4xFf2Z4 +aE1xRBBNd1Ke5C0zToSaYw5o4B/7X99nKK2/XaUX1byLow2aju2XJl2OpKpJg+tSJ2fmjIJTkfuY +Uz574dFX6/VXxSxwGH/xQEAKS5TCsBK3CwnuG1p5SAsQq3gGVozDWyjEBcWDMdy8/AIFrj/y03Lf +c/RNRCQTAfZbnf2QwV7sluw4fH3XJr07UoD0YqN+7XZzidtrwqMY26fpLZnyZjnBEt1FAZWO7RnK +G5asg8xRk9YaDdedXdQSJAOy6bWEWlABj+tVAigBxavaluUH8LOj+yfCFldJjNLdi90fVHkUD/m4 +Mr5OtmupNMXPwuG3EQlqWUVpQoYpUYKLsk7a5Mvg6UFkiH596y5IbJEVCI1Kb3D1`, "e3+wo77iKcILiZegnzyUNcjCdoQ="}, +} + +func TestQuickXorHash(t *testing.T) { + for _, test := range testVectors { + what := fmt.Sprintf("test size %d", test.size) + in, err := base64.StdEncoding.DecodeString(test.in) + require.NoError(t, err, what) + got := Sum(in) + want, err := base64.StdEncoding.DecodeString(test.out) + require.NoError(t, err, what) + assert.Equal(t, want, got[:], what) + } +} + +func TestQuickXorHashByBlock(t *testing.T) { + for _, blockSize := range []int{1, 2, 4, 7, 8, 16, 32, 64, 128, 256, 512} { + for _, test := range testVectors { + what := fmt.Sprintf("test size %d blockSize %d", test.size, blockSize) + in, err := base64.StdEncoding.DecodeString(test.in) + require.NoError(t, err, what) + h := New() + for i := 0; i < len(in); i += blockSize { + end := i + blockSize + if end > len(in) { + end = len(in) + } + n, err := h.Write(in[i:end]) + require.Equal(t, end-i, n, what) + require.NoError(t, err, what) + } + got := h.Sum(nil) + want, err := base64.StdEncoding.DecodeString(test.out) + require.NoError(t, err, what) + assert.Equal(t, want, got, test.size, what) + } + } +} + +func TestSize(t *testing.T) { + d := New() + assert.Equal(t, 20, d.Size()) +} + +func TestBlockSize(t *testing.T) { + d := New() + assert.Equal(t, 64, d.BlockSize()) +} + +func TestReset(t *testing.T) { + d := New() + zeroHash := d.Sum(nil) + _, _ = d.Write([]byte{1}) + assert.NotEqual(t, zeroHash, d.Sum(nil)) + d.Reset() + assert.Equal(t, zeroHash, d.Sum(nil)) +} + +// check interface +var _ hash.Hash = (*quickXorHash)(nil) diff --git a/.rclone_repo/backend/onedrive/replace.go b/.rclone_repo/backend/onedrive/replace.go new file mode 100755 index 0000000..1d38d56 --- /dev/null +++ b/.rclone_repo/backend/onedrive/replace.go @@ -0,0 +1,91 @@ +/* +Translate file names for one drive + +OneDrive reserved characters + +The following characters are OneDrive reserved characters, and can't +be used in OneDrive folder and file names. + + onedrive-reserved = "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|" + onedrive-business-reserved + = "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|" / "#" / "%" + +Note: Folder names can't end with a period (.). + +Note: OneDrive for Business file or folder names cannot begin with a +tilde ('~'). + +*/ + +package onedrive + +import ( + "regexp" + "strings" +) + +// charMap holds replacements for characters +// +// Onedrive has a restricted set of characters compared to other cloud +// storage systems, so we to map these to the FULLWIDTH unicode +// equivalents +// +// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS +var ( + charMap = map[rune]rune{ + '\\': '＼', // FULLWIDTH REVERSE SOLIDUS + '*': '＊', // FULLWIDTH ASTERISK + '<': '＜', // FULLWIDTH LESS-THAN SIGN + '>': '＞', // FULLWIDTH GREATER-THAN SIGN + '?': '？', // FULLWIDTH QUESTION MARK + ':': '：', // FULLWIDTH COLON + '|': '｜', // FULLWIDTH VERTICAL LINE + '#': '＃', // FULLWIDTH NUMBER SIGN + '%': '％', // FULLWIDTH PERCENT SIGN + '"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved + '.': '．', // FULLWIDTH FULL STOP + '~': '～', // FULLWIDTH TILDE + ' ': '␠', // SYMBOL FOR SPACE + } + invCharMap map[rune]rune + fixEndingInPeriod = regexp.MustCompile(`\.(/|$)`) + fixStartingWithTilde = regexp.MustCompile(`(/|^)~`) + fixStartingWithSpace = regexp.MustCompile(`(/|^) `) +) + +func init() { + // Create inverse charMap + invCharMap = make(map[rune]rune, len(charMap)) + for k, v := range charMap { + invCharMap[v] = k + } +} + +// replaceReservedChars takes a path and substitutes any reserved +// characters in it +func replaceReservedChars(in string) string { + // Folder names can't end with a period '.' + in = fixEndingInPeriod.ReplaceAllString(in, string(charMap['.'])+"$1") + // OneDrive for Business file or folder names cannot begin with a tilde '~' + in = fixStartingWithTilde.ReplaceAllString(in, "$1"+string(charMap['~'])) + // Apparently file names can't start with space either + in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' '])) + // Replace reserved characters + return strings.Map(func(c rune) rune { + if replacement, ok := charMap[c]; ok && c != '.' && c != '~' && c != ' ' { + return replacement + } + return c + }, in) +} + +// restoreReservedChars takes a path and undoes any substitutions +// made by replaceReservedChars +func restoreReservedChars(in string) string { + return strings.Map(func(c rune) rune { + if replacement, ok := invCharMap[c]; ok { + return replacement + } + return c + }, in) +} diff --git a/.rclone_repo/backend/onedrive/replace_test.go b/.rclone_repo/backend/onedrive/replace_test.go new file mode 100755 index 0000000..bac8a59 --- /dev/null +++ b/.rclone_repo/backend/onedrive/replace_test.go @@ -0,0 +1,30 @@ +package onedrive + +import "testing" + +func TestReplace(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"", ""}, + {"abc 123", "abc 123"}, + {`\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~`}, + {`\*<>?:|#%".~/\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~/＼＊＜＞？：｜＃％＂.~`}, + {" leading space", "␠leading space"}, + {"~leading tilde", "～leading tilde"}, + {"trailing dot.", "trailing dot．"}, + {" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"}, + {"~leading tilde/~leading tilde/~leading tilde", "～leading tilde/～leading tilde/～leading tilde"}, + {"trailing dot./trailing dot./trailing dot.", "trailing dot．/trailing dot．/trailing dot．"}, + } { + got := replaceReservedChars(test.in) + if got != test.out { + t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got) + } + got2 := restoreReservedChars(got) + if got2 != test.in { + t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2) + } + } +} diff --git a/.rclone_repo/backend/opendrive/opendrive.go b/.rclone_repo/backend/opendrive/opendrive.go new file mode 100755 index 0000000..1d2e820 --- /dev/null +++ b/.rclone_repo/backend/opendrive/opendrive.go @@ -0,0 +1,1117 @@ +package opendrive + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/http" + "path" + "strconv" + "strings" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/dircache" + "github.com/ncw/rclone/lib/pacer" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" +) + +const ( + defaultEndpoint = "https://dev.opendrive.com/api/v1" + minSleep = 10 * time.Millisecond + maxSleep = 5 * time.Minute + decayConstant = 1 // bigger for slower decay, exponential +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "opendrive", + Description: "OpenDrive", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "username", + Help: "Username", + Required: true, + }, { + Name: "password", + Help: "Password.", + IsPassword: true, + Required: true, + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + UserName string `config:"username"` + Password string `config:"password"` +} + +// Fs represents a remote server +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + features *fs.Features // optional features + srv *rest.Client // the connection to the server + pacer *pacer.Pacer // To pace and retry the API calls + session UserSessionInfo // contains the session data + dirCache *dircache.DirCache // Map of directory path to directory id +} + +// Object describes an object +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + id string // ID of the file + modTime time.Time // The modified time of the object if known + md5 string // MD5 hash if known + size int64 // Size of the object +} + +// parsePath parses an incoming 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("OpenDrive root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *Fs) DirCacheFlush() { + f.dirCache.ResetRoot() +} + +// NewFs contstructs an Fs from the path, bucket:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + root = parsePath(root) + if opt.UserName == "" { + return nil, errors.New("username not found") + } + opt.Password, err = obscure.Reveal(opt.Password) + if err != nil { + return nil, errors.New("password could not revealed") + } + if opt.Password == "" { + return nil, errors.New("password not found") + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler), + pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), + } + + f.dirCache = dircache.New(root, "0", f) + + // set the rootURL for the REST client + f.srv.SetRoot(defaultEndpoint) + + // get sessionID + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + account := Account{Username: opt.UserName, Password: opt.Password} + + opts := rest.Opts{ + Method: "POST", + Path: "/session/login.json", + } + resp, err = f.srv.CallJSON(&opts, &account, &f.session) + return f.shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to create session") + } + fs.Debugf(nil, "Starting OpenDrive session with ID: %s", f.session.SessionID) + + f.features = (&fs.Features{ + CaseInsensitive: true, + CanHaveEmptyDirectories: true, + }).Fill(f) + + // Find the current root + err = f.dirCache.FindRoot(false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + newF := *f + newF.dirCache = dircache.New(newRoot, "0", &newF) + newF.root = newRoot + + // Make new Fs which is the parent + err = newF.dirCache.FindRoot(false) + if err != nil { + // No root so return old f + return f, nil + } + _, err := newF.newObjectWithInfo(remote, nil) + if err != nil { + if err == fs.ErrorObjectNotFound { + // File doesn't exist so return old f + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return &newF, fs.ErrorIsFile + } + return f, nil +} + +// rootSlash returns root with a slash on if it is empty, otherwise empty string +func (f *Fs) rootSlash() string { + if f.root == "" { + return f.root + } + return f.root + "/" +} + +// errorHandler parses a non 2xx error response into an error +func errorHandler(resp *http.Response) error { + errResponse := new(Error) + err := rest.DecodeJSON(resp, &errResponse) + if err != nil { + fs.Debugf(nil, "Couldn't decode error response: %v", err) + } + if errResponse.Info.Code == 0 { + errResponse.Info.Code = resp.StatusCode + } + if errResponse.Info.Message == "" { + errResponse.Info.Message = "Unknown " + resp.Status + } + return errResponse +} + +// Mkdir creates the folder if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + // fs.Debugf(nil, "Mkdir(\"%s\")", dir) + err := f.dirCache.FindRoot(true) + if err != nil { + return err + } + if dir != "" { + _, err = f.dirCache.FindDir(dir, true) + } + return err +} + +// deleteObject removes an object by ID +func (f *Fs) deleteObject(id string) error { + return f.pacer.Call(func() (bool, error) { + removeDirData := removeFolder{SessionID: f.session.SessionID, FolderID: id} + opts := rest.Opts{ + Method: "POST", + NoResponse: true, + Path: "/folder/remove.json", + } + resp, err := f.srv.CallJSON(&opts, &removeDirData, nil) + return f.shouldRetry(resp, err) + }) +} + +// purgeCheck remotes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(dir string, check bool) error { + root := path.Join(f.root, dir) + if root == "" { + return errors.New("can't purge root directory") + } + dc := f.dirCache + err := dc.FindRoot(false) + if err != nil { + return err + } + rootID, err := dc.FindDir(dir, false) + if err != nil { + return err + } + item, err := f.readMetaDataForFolderID(rootID) + if err != nil { + return err + } + if check && len(item.Files) != 0 { + return errors.New("folder not empty") + } + err = f.deleteObject(rootID) + if err != nil { + return err + } + f.dirCache.FlushDir(dir) + if err != nil { + return err + } + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + // fs.Debugf(nil, "Rmdir(\"%s\")", path.Join(f.root, dir)) + return f.purgeCheck(dir, true) +} + +// Precision of the remote +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + // fs.Debugf(nil, "Copy(%v)", remote) + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + err := srcObj.readMetaData() + if err != nil { + return nil, err + } + + srcPath := srcObj.fs.rootSlash() + srcObj.remote + dstPath := f.rootSlash() + remote + if strings.ToLower(srcPath) == strings.ToLower(dstPath) { + return nil, errors.Errorf("Can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + // fs.Debugf(nil, "...%#v\n...%#v", remote, directoryID) + + // Copy the object + var resp *http.Response + response := moveCopyFileResponse{} + err = f.pacer.Call(func() (bool, error) { + copyFileData := moveCopyFile{ + SessionID: f.session.SessionID, + SrcFileID: srcObj.id, + DstFolderID: directoryID, + Move: "false", + OverwriteIfExists: "true", + NewFileName: leaf, + } + opts := rest.Opts{ + Method: "POST", + Path: "/file/move_copy.json", + } + resp, err = f.srv.CallJSON(&opts, ©FileData, &response) + return f.shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + + size, _ := strconv.ParseInt(response.Size, 10, 64) + dstObj.id = response.FileID + dstObj.size = size + + return dstObj, nil +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + // fs.Debugf(nil, "Move(%v)", remote) + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantCopy + } + err := srcObj.readMetaData() + if err != nil { + return nil, err + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + // Copy the object + var resp *http.Response + response := moveCopyFileResponse{} + err = f.pacer.Call(func() (bool, error) { + copyFileData := moveCopyFile{ + SessionID: f.session.SessionID, + SrcFileID: srcObj.id, + DstFolderID: directoryID, + Move: "true", + OverwriteIfExists: "true", + NewFileName: leaf, + } + opts := rest.Opts{ + Method: "POST", + Path: "/file/move_copy.json", + } + resp, err = f.srv.CallJSON(&opts, ©FileData, &response) + return f.shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + + size, _ := strconv.ParseInt(response.Size, 10, 64) + dstObj.id = response.FileID + dstObj.size = size + + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) (err error) { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := path.Join(srcFs.root, srcRemote) + dstPath := path.Join(f.root, dstRemote) + + // Refuse to move to or from the root + if srcPath == "" || dstPath == "" { + fs.Debugf(src, "DirMove error: Can't move root") + return errors.New("can't move root directory") + } + + // find the root src directory + err = srcFs.dirCache.FindRoot(false) + if err != nil { + return err + } + + // find the root dst directory + if dstRemote != "" { + err = f.dirCache.FindRoot(true) + if err != nil { + return err + } + } else { + if f.dirCache.FoundRoot() { + return fs.ErrorDirExists + } + } + + // Find ID of dst parent, creating subdirs if necessary + var leaf, directoryID string + findPath := dstRemote + if dstRemote == "" { + findPath = f.root + } + leaf, directoryID, err = f.dirCache.FindPath(findPath, true) + if err != nil { + return err + } + + // Check destination does not exist + if dstRemote != "" { + _, err = f.dirCache.FindDir(dstRemote, false) + if err == fs.ErrorDirNotFound { + // OK + } else if err != nil { + return err + } else { + return fs.ErrorDirExists + } + } + + // Find ID of src + srcID, err := srcFs.dirCache.FindDir(srcRemote, false) + if err != nil { + return err + } + + // Do the move + var resp *http.Response + response := moveCopyFolderResponse{} + err = f.pacer.Call(func() (bool, error) { + moveFolderData := moveCopyFolder{ + SessionID: f.session.SessionID, + FolderID: srcID, + DstFolderID: directoryID, + Move: "true", + NewFolderName: leaf, + } + opts := rest.Opts{ + Method: "POST", + Path: "/folder/move_copy.json", + } + resp, err = f.srv.CallJSON(&opts, &moveFolderData, &response) + return f.shouldRetry(resp, err) + }) + if err != nil { + fs.Debugf(src, "DirMove error %v", err) + return err + } + + srcFs.dirCache.FlushDir(srcRemote) + return nil +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + return f.purgeCheck("", false) +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, file *File) (fs.Object, error) { + // fs.Debugf(nil, "newObjectWithInfo(%s, %v)", remote, file) + + var o *Object + if nil != file { + o = &Object{ + fs: f, + remote: remote, + id: file.FileID, + modTime: time.Unix(file.DateModified, 0), + size: file.Size, + md5: file.FileHash, + } + } else { + o = &Object{ + fs: f, + remote: remote, + } + + err := o.readMetaData() + if err != nil { + return nil, err + } + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + // fs.Debugf(nil, "NewObject(\"%s\")", remote) + return f.newObjectWithInfo(remote, nil) +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Returns the object, leaf, directoryID and error +// +// Used to create new objects +func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { + // Create the directory for the object if it doesn't exist + leaf, directoryID, err = f.dirCache.FindRootAndPath(remote, true) + if err != nil { + return nil, leaf, directoryID, err + } + // fs.Debugf(nil, "\n...leaf %#v\n...id %#v", leaf, directoryID) + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + } + return o, leaf, directoryID, nil +} + +// readMetaDataForPath reads the metadata from the path +func (f *Fs) readMetaDataForFolderID(id string) (info *FolderList, err error) { + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + Path: "/folder/list.json/" + f.session.SessionID + "/" + id, + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &info) + return f.shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + if resp != nil { + } + + return info, err +} + +// Put the object into the bucket +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime() + + // fs.Debugf(nil, "Put(%s)", remote) + + o, leaf, directoryID, err := f.createObject(remote, modTime, size) + if err != nil { + return nil, err + } + + if "" == o.id { + // Attempt to read ID, ignore error + // FIXME is this correct? + _ = o.readMetaData() + } + + if "" == o.id { + // We need to create a ID for this file + var resp *http.Response + response := createFileResponse{} + err := o.fs.pacer.Call(func() (bool, error) { + createFileData := createFile{SessionID: o.fs.session.SessionID, FolderID: directoryID, Name: replaceReservedChars(leaf)} + opts := rest.Opts{ + Method: "POST", + Path: "/upload/create_file.json", + } + resp, err = o.fs.srv.CallJSON(&opts, &createFileData, &response) + return o.fs.shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to create file") + } + + o.id = response.FileID + } + + return o, o.Update(in, src, options...) +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 400, // Bad request (seen in "Next token is expired") + 401, // Unauthorized (seen in "Token has expired") + 408, // Request Timeout + 423, // Locked - get this on folders sometimes + 429, // Rate exceeded. + 500, // Get occasional 500 Internal Server Error + 502, // Bad Gateway when doing big listings + 503, // Service Unavailable + 504, // Gateway Time-out +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// DirCacher methods + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) { + // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, replaceReservedChars(leaf)) + var resp *http.Response + response := createFolderResponse{} + err = f.pacer.Call(func() (bool, error) { + createDirData := createFolder{ + SessionID: f.session.SessionID, + FolderName: replaceReservedChars(leaf), + FolderSubParent: pathID, + FolderIsPublic: 0, + FolderPublicUpl: 0, + FolderPublicDisplay: 0, + FolderPublicDnl: 0, + } + opts := rest.Opts{ + Method: "POST", + Path: "/folder.json", + } + resp, err = f.srv.CallJSON(&opts, &createDirData, &response) + return f.shouldRetry(resp, err) + }) + if err != nil { + return "", err + } + + return response.FolderID, nil +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) { + // fs.Debugf(nil, "FindLeaf(\"%s\", \"%s\")", pathID, leaf) + + if pathID == "0" && leaf == "" { + // fs.Debugf(nil, "Found OpenDrive root") + // that's the root directory + return pathID, true, nil + } + + // get the folderIDs + var resp *http.Response + folderList := FolderList{} + err = f.pacer.Call(func() (bool, error) { + opts := rest.Opts{ + Method: "GET", + Path: "/folder/list.json/" + f.session.SessionID + "/" + pathID, + } + resp, err = f.srv.CallJSON(&opts, nil, &folderList) + return f.shouldRetry(resp, err) + }) + if err != nil { + return "", false, errors.Wrap(err, "failed to get folder list") + } + + for _, folder := range folderList.Folders { + folder.Name = restoreReservedChars(folder.Name) + // fs.Debugf(nil, "Folder: %s (%s)", folder.Name, folder.FolderID) + + if leaf == folder.Name { + // found + return folder.FolderID, true, nil + } + } + + return "", false, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + // fs.Debugf(nil, "List(%v)", dir) + err = f.dirCache.FindRoot(false) + if err != nil { + return nil, err + } + directoryID, err := f.dirCache.FindDir(dir, false) + if err != nil { + return nil, err + } + + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + Path: "/folder/list.json/" + f.session.SessionID + "/" + directoryID, + } + folderList := FolderList{} + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &folderList) + return f.shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to get folder list") + } + + for _, folder := range folderList.Folders { + folder.Name = restoreReservedChars(folder.Name) + // fs.Debugf(nil, "Folder: %s (%s)", folder.Name, folder.FolderID) + remote := path.Join(dir, folder.Name) + // cache the directory ID for later lookups + f.dirCache.Put(remote, folder.FolderID) + d := fs.NewDir(remote, time.Unix(int64(folder.DateModified), 0)).SetID(folder.FolderID) + d.SetItems(int64(folder.ChildFolders)) + entries = append(entries, d) + } + + for _, file := range folderList.Files { + file.Name = restoreReservedChars(file.Name) + // fs.Debugf(nil, "File: %s (%s)", file.Name, file.FileID) + remote := path.Join(dir, file.Name) + o, err := f.newObjectWithInfo(remote, &file) + if err != nil { + return nil, err + } + entries = append(entries, o) + } + + return entries, nil +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the Md5sum of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.MD5 { + return "", hash.ErrUnsupported + } + return o.md5, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.size // Object is likely PENDING +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + return o.modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + // fs.Debugf(nil, "SetModTime(%v)", modTime.String()) + opts := rest.Opts{ + Method: "PUT", + NoResponse: true, + Path: "/file/filesettings.json", + } + update := modTimeFile{SessionID: o.fs.session.SessionID, FileID: o.id, FileModificationTime: strconv.FormatInt(modTime.Unix(), 10)} + err := o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.CallJSON(&opts, &update, nil) + return o.fs.shouldRetry(resp, err) + }) + + o.modTime = modTime + + return err +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + // fs.Debugf(nil, "Open(\"%v\")", o.remote) + fs.FixRangeOption(options, o.size) + opts := rest.Opts{ + Method: "GET", + Path: "/download/file.json/" + o.id + "?session_id=" + o.fs.session.SessionID, + Options: options, + } + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return o.fs.shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to open file)") + } + + return resp.Body, nil +} + +// Remove an object +func (o *Object) Remove() error { + // fs.Debugf(nil, "Remove(\"%s\")", o.id) + return o.fs.pacer.Call(func() (bool, error) { + opts := rest.Opts{ + Method: "DELETE", + NoResponse: true, + Path: "/file.json/" + o.fs.session.SessionID + "/" + o.id, + } + resp, err := o.fs.srv.Call(&opts) + return o.fs.shouldRetry(resp, err) + }) +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + size := src.Size() + modTime := src.ModTime() + // fs.Debugf(nil, "Update(\"%s\", \"%s\")", o.id, o.remote) + + // Open file for upload + var resp *http.Response + openResponse := openUploadResponse{} + err := o.fs.pacer.Call(func() (bool, error) { + openUploadData := openUpload{SessionID: o.fs.session.SessionID, FileID: o.id, Size: size} + // fs.Debugf(nil, "PreOpen: %#v", openUploadData) + opts := rest.Opts{ + Method: "POST", + Path: "/upload/open_file_upload.json", + } + resp, err := o.fs.srv.CallJSON(&opts, &openUploadData, &openResponse) + return o.fs.shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "failed to create file") + } + // resp.Body.Close() + // fs.Debugf(nil, "PostOpen: %#v", openResponse) + + // 1 MB chunks size + chunkSize := int64(1024 * 1024 * 10) + chunkOffset := int64(0) + remainingBytes := size + chunkCounter := 0 + + for remainingBytes > 0 { + currentChunkSize := chunkSize + if currentChunkSize > remainingBytes { + currentChunkSize = remainingBytes + } + remainingBytes -= currentChunkSize + fs.Debugf(o, "Uploading chunk %d, size=%d, remain=%d", chunkCounter, currentChunkSize, remainingBytes) + + err = o.fs.pacer.Call(func() (bool, error) { + var formBody bytes.Buffer + w := multipart.NewWriter(&formBody) + fw, err := w.CreateFormFile("file_data", o.remote) + if err != nil { + return false, err + } + if _, err = io.CopyN(fw, in, currentChunkSize); err != nil { + return false, err + } + // Add session_id + if fw, err = w.CreateFormField("session_id"); err != nil { + return false, err + } + if _, err = fw.Write([]byte(o.fs.session.SessionID)); err != nil { + return false, err + } + // Add session_id + if fw, err = w.CreateFormField("session_id"); err != nil { + return false, err + } + if _, err = fw.Write([]byte(o.fs.session.SessionID)); err != nil { + return false, err + } + // Add file_id + if fw, err = w.CreateFormField("file_id"); err != nil { + return false, err + } + if _, err = fw.Write([]byte(o.id)); err != nil { + return false, err + } + // Add temp_location + if fw, err = w.CreateFormField("temp_location"); err != nil { + return false, err + } + if _, err = fw.Write([]byte(openResponse.TempLocation)); err != nil { + return false, err + } + // Add chunk_offset + if fw, err = w.CreateFormField("chunk_offset"); err != nil { + return false, err + } + if _, err = fw.Write([]byte(strconv.FormatInt(chunkOffset, 10))); err != nil { + return false, err + } + // Add chunk_size + if fw, err = w.CreateFormField("chunk_size"); err != nil { + return false, err + } + if _, err = fw.Write([]byte(strconv.FormatInt(currentChunkSize, 10))); err != nil { + return false, err + } + // Don't forget to close the multipart writer. + // If you don't close it, your request will be missing the terminating boundary. + err = w.Close() + if err != nil { + return false, err + } + + opts := rest.Opts{ + Method: "POST", + Path: "/upload/upload_file_chunk.json", + Body: &formBody, + ExtraHeaders: map[string]string{"Content-Type": w.FormDataContentType()}, + } + resp, err = o.fs.srv.Call(&opts) + return o.fs.shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "failed to create file") + } + err = resp.Body.Close() + if err != nil { + return errors.Wrap(err, "close failed on create file") + } + + chunkCounter++ + chunkOffset += currentChunkSize + } + + // Close file for upload + closeResponse := closeUploadResponse{} + err = o.fs.pacer.Call(func() (bool, error) { + closeUploadData := closeUpload{SessionID: o.fs.session.SessionID, FileID: o.id, Size: size, TempLocation: openResponse.TempLocation} + // fs.Debugf(nil, "PreClose: %#v", closeUploadData) + opts := rest.Opts{ + Method: "POST", + Path: "/upload/close_file_upload.json", + } + resp, err = o.fs.srv.CallJSON(&opts, &closeUploadData, &closeResponse) + return o.fs.shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "failed to create file") + } + // fs.Debugf(nil, "PostClose: %#v", closeResponse) + + o.id = closeResponse.FileID + o.size = closeResponse.Size + + // Set the mod time now + err = o.SetModTime(modTime) + if err != nil { + return err + } + + // Set permissions + err = o.fs.pacer.Call(func() (bool, error) { + update := permissions{SessionID: o.fs.session.SessionID, FileID: o.id, FileIsPublic: 0} + // fs.Debugf(nil, "Permissions : %#v", update) + opts := rest.Opts{ + Method: "POST", + NoResponse: true, + Path: "/file/access.json", + } + resp, err = o.fs.srv.CallJSON(&opts, &update, nil) + return o.fs.shouldRetry(resp, err) + }) + if err != nil { + return err + } + + return o.readMetaData() +} + +func (o *Object) readMetaData() (err error) { + leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(o.remote, false) + if err != nil { + if err == fs.ErrorDirNotFound { + return fs.ErrorObjectNotFound + } + return err + } + var resp *http.Response + folderList := FolderList{} + err = o.fs.pacer.Call(func() (bool, error) { + opts := rest.Opts{ + Method: "GET", + Path: "/folder/itembyname.json/" + o.fs.session.SessionID + "/" + directoryID + "?name=" + rest.URLPathEscape(replaceReservedChars(leaf)), + } + resp, err = o.fs.srv.CallJSON(&opts, nil, &folderList) + return o.fs.shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "failed to get folder list") + } + + if len(folderList.Files) == 0 { + return fs.ErrorObjectNotFound + } + + leafFile := folderList.Files[0] + o.id = leafFile.FileID + o.modTime = time.Unix(leafFile.DateModified, 0) + o.md5 = leafFile.FileHash + o.size = leafFile.Size + + return nil +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.id +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.IDer = (*Object)(nil) +) diff --git a/.rclone_repo/backend/opendrive/opendrive_test.go b/.rclone_repo/backend/opendrive/opendrive_test.go new file mode 100755 index 0000000..66a0a35 --- /dev/null +++ b/.rclone_repo/backend/opendrive/opendrive_test.go @@ -0,0 +1,17 @@ +// Test Opendrive filesystem interface +package opendrive_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/opendrive" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestOpenDrive:", + NilObject: (*opendrive.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/opendrive/replace.go b/.rclone_repo/backend/opendrive/replace.go new file mode 100755 index 0000000..15dcef5 --- /dev/null +++ b/.rclone_repo/backend/opendrive/replace.go @@ -0,0 +1,78 @@ +/* +Translate file names for OpenDrive + +OpenDrive reserved characters + +The following characters are OpenDrive reserved characters, and can't +be used in OpenDrive folder and file names. + +\ / : * ? " < > | + +OpenDrive files and folders can't have leading or trailing spaces also. + +*/ + +package opendrive + +import ( + "regexp" + "strings" +) + +// charMap holds replacements for characters +// +// OpenDrive has a restricted set of characters compared to other cloud +// storage systems, so we to map these to the FULLWIDTH unicode +// equivalents +// +// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS +var ( + charMap = map[rune]rune{ + '\\': '＼', // FULLWIDTH REVERSE SOLIDUS + ':': '：', // FULLWIDTH COLON + '*': '＊', // FULLWIDTH ASTERISK + '?': '？', // FULLWIDTH QUESTION MARK + '"': '＂', // FULLWIDTH QUOTATION MARK + '<': '＜', // FULLWIDTH LESS-THAN SIGN + '>': '＞', // FULLWIDTH GREATER-THAN SIGN + '|': '｜', // FULLWIDTH VERTICAL LINE + ' ': '␠', // SYMBOL FOR SPACE + } + fixStartingWithSpace = regexp.MustCompile(`(/|^) `) + fixEndingWithSpace = regexp.MustCompile(` (/|$)`) + invCharMap map[rune]rune +) + +func init() { + // Create inverse charMap + invCharMap = make(map[rune]rune, len(charMap)) + for k, v := range charMap { + invCharMap[v] = k + } +} + +// replaceReservedChars takes a path and substitutes any reserved +// characters in it +func replaceReservedChars(in string) string { + // Filenames can't start with space + in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' '])) + // Filenames can't end with space + in = fixEndingWithSpace.ReplaceAllString(in, string(charMap[' '])+"$1") + return strings.Map(func(c rune) rune { + if replacement, ok := charMap[c]; ok && c != ' ' { + return replacement + } + return c + }, in) +} + +// restoreReservedChars takes a path and undoes any substitutions +// made by replaceReservedChars +func restoreReservedChars(in string) string { + return strings.Map(func(c rune) rune { + if replacement, ok := invCharMap[c]; ok { + return replacement + } + return c + }, in) +} diff --git a/.rclone_repo/backend/opendrive/replace_test.go b/.rclone_repo/backend/opendrive/replace_test.go new file mode 100755 index 0000000..1c4978f --- /dev/null +++ b/.rclone_repo/backend/opendrive/replace_test.go @@ -0,0 +1,28 @@ +package opendrive + +import "testing" + +func TestReplace(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"", ""}, + {"abc 123", "abc 123"}, + {`\*<>?:|#%".~`, `＼＊＜＞？：｜#%＂.~`}, + {`\*<>?:|#%".~/\*<>?:|#%".~`, `＼＊＜＞？：｜#%＂.~/＼＊＜＞？：｜#%＂.~`}, + {" leading space", "␠leading space"}, + {" path/ leading spaces", "␠path/␠ leading spaces"}, + {"trailing space ", "trailing space␠"}, + {"trailing spaces /path ", "trailing spaces ␠/path␠"}, + } { + got := replaceReservedChars(test.in) + if got != test.out { + t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got) + } + got2 := restoreReservedChars(got) + if got2 != test.in { + t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2) + } + } +} diff --git a/.rclone_repo/backend/opendrive/types.go b/.rclone_repo/backend/opendrive/types.go new file mode 100755 index 0000000..1bd7328 --- /dev/null +++ b/.rclone_repo/backend/opendrive/types.go @@ -0,0 +1,214 @@ +package opendrive + +import ( + "encoding/json" + "fmt" +) + +// Error describes an openDRIVE error response +type Error struct { + Info struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` +} + +// Error statisfies the error interface +func (e *Error) Error() string { + return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code) +} + +// Account describes a OpenDRIVE account +type Account struct { + Username string `json:"username"` + Password string `json:"passwd"` +} + +// UserSessionInfo describes a OpenDRIVE session +type UserSessionInfo struct { + Username string `json:"username"` + Password string `json:"passwd"` + + SessionID string `json:"SessionID"` + UserName string `json:"UserName"` + UserFirstName string `json:"UserFirstName"` + UserLastName string `json:"UserLastName"` + AccType string `json:"AccType"` + UserLang string `json:"UserLang"` + UserID string `json:"UserID"` + IsAccountUser json.RawMessage `json:"IsAccountUser"` + DriveName string `json:"DriveName"` + UserLevel string `json:"UserLevel"` + UserPlan string `json:"UserPlan"` + FVersioning string `json:"FVersioning"` + UserDomain string `json:"UserDomain"` + PartnerUsersDomain string `json:"PartnerUsersDomain"` +} + +// FolderList describes a OpenDRIVE listing +type FolderList struct { + // DirUpdateTime string `json:"DirUpdateTime,string"` + Name string `json:"Name"` + ParentFolderID string `json:"ParentFolderID"` + DirectFolderLink string `json:"DirectFolderLink"` + ResponseType int `json:"ResponseType"` + Folders []Folder `json:"Folders"` + Files []File `json:"Files"` +} + +// Folder describes a OpenDRIVE folder +type Folder struct { + FolderID string `json:"FolderID"` + Name string `json:"Name"` + DateCreated int `json:"DateCreated"` + DirUpdateTime int `json:"DirUpdateTime"` + Access int `json:"Access"` + DateModified int64 `json:"DateModified"` + Shared string `json:"Shared"` + ChildFolders int `json:"ChildFolders"` + Link string `json:"Link"` + Encrypted string `json:"Encrypted"` +} + +type createFolder struct { + SessionID string `json:"session_id"` + FolderName string `json:"folder_name"` + FolderSubParent string `json:"folder_sub_parent"` + FolderIsPublic int64 `json:"folder_is_public"` // (0 = private, 1 = public, 2 = hidden) + FolderPublicUpl int64 `json:"folder_public_upl"` // (0 = disabled, 1 = enabled) + FolderPublicDisplay int64 `json:"folder_public_display"` // (0 = disabled, 1 = enabled) + FolderPublicDnl int64 `json:"folder_public_dnl"` // (0 = disabled, 1 = enabled). +} + +type createFolderResponse struct { + FolderID string `json:"FolderID"` + Name string `json:"Name"` + DateCreated int `json:"DateCreated"` + DirUpdateTime int `json:"DirUpdateTime"` + Access int `json:"Access"` + DateModified int `json:"DateModified"` + Shared string `json:"Shared"` + Description string `json:"Description"` + Link string `json:"Link"` +} + +type moveCopyFolder struct { + SessionID string `json:"session_id"` + FolderID string `json:"folder_id"` + DstFolderID string `json:"dst_folder_id"` + Move string `json:"move"` + NewFolderName string `json:"new_folder_name"` // New name for destination folder. +} + +type moveCopyFolderResponse struct { + FolderID string `json:"FolderID"` +} + +type removeFolder struct { + SessionID string `json:"session_id"` + FolderID string `json:"folder_id"` +} + +// File describes a OpenDRIVE file +type File struct { + FileID string `json:"FileId"` + FileHash string `json:"FileHash"` + Name string `json:"Name"` + GroupID int `json:"GroupID"` + Extension string `json:"Extension"` + Size int64 `json:"Size,string"` + Views string `json:"Views"` + Version string `json:"Version"` + Downloads string `json:"Downloads"` + DateModified int64 `json:"DateModified,string"` + Access string `json:"Access"` + Link string `json:"Link"` + DownloadLink string `json:"DownloadLink"` + StreamingLink string `json:"StreamingLink"` + TempStreamingLink string `json:"TempStreamingLink"` + EditLink string `json:"EditLink"` + ThumbLink string `json:"ThumbLink"` + Password string `json:"Password"` + EditOnline int `json:"EditOnline"` +} + +type moveCopyFile struct { + SessionID string `json:"session_id"` + SrcFileID string `json:"src_file_id"` + DstFolderID string `json:"dst_folder_id"` + Move string `json:"move"` + OverwriteIfExists string `json:"overwrite_if_exists"` + NewFileName string `json:"new_file_name"` // New name for destination file. +} + +type moveCopyFileResponse struct { + FileID string `json:"FileID"` + Size string `json:"Size"` +} + +type createFile struct { + SessionID string `json:"session_id"` + FolderID string `json:"folder_id"` + Name string `json:"file_name"` +} + +type createFileResponse struct { + FileID string `json:"FileId"` + Name string `json:"Name"` + GroupID int `json:"GroupID"` + Extension string `json:"Extension"` + Size string `json:"Size"` + Views string `json:"Views"` + Downloads string `json:"Downloads"` + DateModified string `json:"DateModified"` + Access string `json:"Access"` + Link string `json:"Link"` + DownloadLink string `json:"DownloadLink"` + StreamingLink string `json:"StreamingLink"` + TempStreamingLink string `json:"TempStreamingLink"` + DirUpdateTime int `json:"DirUpdateTime"` + TempLocation string `json:"TempLocation"` + SpeedLimit int `json:"SpeedLimit"` + RequireCompression int `json:"RequireCompression"` + RequireHash int `json:"RequireHash"` + RequireHashOnly int `json:"RequireHashOnly"` +} + +type modTimeFile struct { + SessionID string `json:"session_id"` + FileID string `json:"file_id"` + FileModificationTime string `json:"file_modification_time"` +} + +type openUpload struct { + SessionID string `json:"session_id"` + FileID string `json:"file_id"` + Size int64 `json:"file_size"` +} + +type openUploadResponse struct { + TempLocation string `json:"TempLocation"` + RequireCompression bool `json:"RequireCompression"` + RequireHash bool `json:"RequireHash"` + RequireHashOnly bool `json:"RequireHashOnly"` + SpeedLimit int `json:"SpeedLimit"` +} + +type closeUpload struct { + SessionID string `json:"session_id"` + FileID string `json:"file_id"` + Size int64 `json:"file_size"` + TempLocation string `json:"temp_location"` +} + +type closeUploadResponse struct { + FileID string `json:"FileID"` + FileHash string `json:"FileHash"` + Size int64 `json:"Size"` +} + +type permissions struct { + SessionID string `json:"session_id"` + FileID string `json:"file_id"` + FileIsPublic int64 `json:"file_ispublic"` +} diff --git a/.rclone_repo/backend/pcloud/api/types.go b/.rclone_repo/backend/pcloud/api/types.go new file mode 100755 index 0000000..408be79 --- /dev/null +++ b/.rclone_repo/backend/pcloud/api/types.go @@ -0,0 +1,185 @@ +// Package api has type definitions for pcloud +// +// Converted from the API docs with help from https://mholt.github.io/json-to-go/ +package api + +import ( + "fmt" + "time" +) + +const ( + // Sun, 16 Mar 2014 17:26:04 +0000 + timeFormat = `"` + time.RFC1123Z + `"` +) + +// Time represents represents date and time information for the +// pcloud API, by using RFC1123Z +type Time time.Time + +// MarshalJSON turns a Time into JSON (in UTC) +func (t *Time) MarshalJSON() (out []byte, err error) { + timeString := (*time.Time)(t).Format(timeFormat) + return []byte(timeString), nil +} + +// UnmarshalJSON turns JSON into a Time +func (t *Time) UnmarshalJSON(data []byte) error { + newT, err := time.Parse(timeFormat, string(data)) + if err != nil { + return err + } + *t = Time(newT) + return nil +} + +// Error is returned from pcloud when things go wrong +// +// If result is 0 then everything is OK +type Error struct { + Result int `json:"result"` + ErrorString string `json:"error"` +} + +// Error returns a string for the error and statistifes the error interface +func (e *Error) Error() string { + return fmt.Sprintf("pcloud error: %s (%d)", e.ErrorString, e.Result) +} + +// Update returns err directly if it was != nil, otherwise it returns +// an Error or nil if no error was detected +func (e *Error) Update(err error) error { + if err != nil { + return err + } + if e.Result == 0 { + return nil + } + return e +} + +// Check Error statisfies the error interface +var _ error = (*Error)(nil) + +// Item describes a folder or a file as returned by Get Folder Items and others +type Item struct { + Path string `json:"path"` + Name string `json:"name"` + Created Time `json:"created"` + IsMine bool `json:"ismine"` + Thumb bool `json:"thumb"` + Modified Time `json:"modified"` + Comments int `json:"comments"` + ID string `json:"id"` + IsShared bool `json:"isshared"` + IsDeleted bool `json:"isdeleted"` + Icon string `json:"icon"` + IsFolder bool `json:"isfolder"` + ParentFolderID int64 `json:"parentfolderid"` + FolderID int64 `json:"folderid,omitempty"` + Height int `json:"height,omitempty"` + FileID int64 `json:"fileid,omitempty"` + Width int `json:"width,omitempty"` + Hash uint64 `json:"hash,omitempty"` + Category int `json:"category,omitempty"` + Size int64 `json:"size,omitempty"` + ContentType string `json:"contenttype,omitempty"` + Contents []Item `json:"contents"` +} + +// ModTime returns the modification time of the item +func (i *Item) ModTime() (t time.Time) { + t = time.Time(i.Modified) + if t.IsZero() { + t = time.Time(i.Created) + } + return t +} + +// ItemResult is returned from the /listfolder, /createfolder, /deletefolder, /deletefile etc methods +type ItemResult struct { + Error + Metadata Item `json:"metadata"` +} + +// Hashes contains the supported hashes +type Hashes struct { + SHA1 string `json:"sha1"` + MD5 string `json:"md5"` +} + +// UploadFileResponse is the response from /uploadfile +type UploadFileResponse struct { + Error + Items []Item `json:"metadata"` + Checksums []Hashes `json:"checksums"` + Fileids []int64 `json:"fileids"` +} + +// GetFileLinkResult is returned from /getfilelink +type GetFileLinkResult struct { + Error + Dwltag string `json:"dwltag"` + Hash uint64 `json:"hash"` + Size int64 `json:"size"` + Expires Time `json:"expires"` + Path string `json:"path"` + Hosts []string `json:"hosts"` +} + +// IsValid returns whether the link is valid and has not expired +func (g *GetFileLinkResult) IsValid() bool { + if g == nil { + return false + } + if len(g.Hosts) == 0 { + return false + } + return time.Time(g.Expires).Sub(time.Now()) > 30*time.Second +} + +// URL returns a URL from the Path and Hosts. Check with IsValid +// before calling. +func (g *GetFileLinkResult) URL() string { + // FIXME rotate the hosts? + return "https://" + g.Hosts[0] + g.Path +} + +// ChecksumFileResult is returned from /checksumfile +type ChecksumFileResult struct { + Error + Hashes + Metadata Item `json:"metadata"` +} + +// UserInfo is returned from /userinfo +type UserInfo struct { + Error + Cryptosetup bool `json:"cryptosetup"` + Plan int `json:"plan"` + CryptoSubscription bool `json:"cryptosubscription"` + PublicLinkQuota int64 `json:"publiclinkquota"` + Email string `json:"email"` + UserID int `json:"userid"` + Result int `json:"result"` + Quota int64 `json:"quota"` + TrashRevretentionDays int `json:"trashrevretentiondays"` + Premium bool `json:"premium"` + PremiumLifetime bool `json:"premiumlifetime"` + EmailVerified bool `json:"emailverified"` + UsedQuota int64 `json:"usedquota"` + Language string `json:"language"` + Business bool `json:"business"` + CryptoLifetime bool `json:"cryptolifetime"` + Registered string `json:"registered"` + Journey struct { + Claimed bool `json:"claimed"` + Steps struct { + VerifyMail bool `json:"verifymail"` + UploadFile bool `json:"uploadfile"` + AutoUpload bool `json:"autoupload"` + DownloadApp bool `json:"downloadapp"` + DownloadDrive bool `json:"downloaddrive"` + } `json:"steps"` + } `json:"journey"` +} diff --git a/.rclone_repo/backend/pcloud/pcloud.go b/.rclone_repo/backend/pcloud/pcloud.go new file mode 100755 index 0000000..ec390e5 --- /dev/null +++ b/.rclone_repo/backend/pcloud/pcloud.go @@ -0,0 +1,1158 @@ +// Package pcloud provides an interface to the Pcloud +// object storage system. +package pcloud + +// FIXME implement ListR? /listfolder can do recursive lists + +// FIXME cleanup returns login required? + +// FIXME mime type? Fix overview if implement. + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/ncw/rclone/backend/pcloud/api" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/dircache" + "github.com/ncw/rclone/lib/oauthutil" + "github.com/ncw/rclone/lib/pacer" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" + "golang.org/x/oauth2" +) + +const ( + rcloneClientID = "DnONSzyJXpm" + rcloneEncryptedClientSecret = "ej1OIF39VOQQ0PXaSdK9ztkLw3tdLNscW2157TKNQdQKkICR4uU7aFg4eFM" + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential + rootID = "d0" // ID of root folder is always this + rootURL = "https://api.pcloud.com" +) + +// Globals +var ( + // Description of how to auth for this app + oauthConfig = &oauth2.Config{ + Scopes: nil, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://my.pcloud.com/oauth2/authorize", + TokenURL: "https://api.pcloud.com/oauth2_token", + }, + ClientID: rcloneClientID, + ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), + RedirectURL: oauthutil.RedirectLocalhostURL, + } +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "pcloud", + Description: "Pcloud", + NewFs: NewFs, + Config: func(name string, m configmap.Mapper) { + err := oauthutil.Config("pcloud", name, m, oauthConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + }, + Options: []fs.Option{{ + Name: config.ConfigClientID, + Help: "Pcloud App Client Id\nLeave blank normally.", + }, { + Name: config.ConfigClientSecret, + Help: "Pcloud App Client Secret\nLeave blank normally.", + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { +} + +// Fs represents a remote pcloud +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + features *fs.Features // optional features + srv *rest.Client // the connection to the server + dirCache *dircache.DirCache // Map of directory path to directory id + pacer *pacer.Pacer // pacer for API calls + tokenRenewer *oauthutil.Renew // renew the token on expiry +} + +// Object describes a pcloud object +// +// Will definitely have info but maybe not meta +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + hasMetaData bool // whether info below has been set + size int64 // size of the object + modTime time.Time // modification time of the object + id string // ID of the object + md5 string // MD5 if known + sha1 string // SHA1 if known + link *api.GetFileLinkResult +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("pcloud root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// parsePath parses an pcloud 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(resp *http.Response, err error) (bool, error) { + doRetry := false + + // Check if it is an api.Error + if apiErr, ok := err.(*api.Error); ok { + // See https://docs.pcloud.com/errors/ for error treatment + // Errors are classified as 1xxx, 2xxx etc + switch apiErr.Result / 1000 { + case 4: // 4xxx: rate limiting + doRetry = true + case 5: // 5xxx: internal errors + doRetry = true + } + } + + if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { + doRetry = true + fs.Debugf(nil, "Should retry: %v", err) + } + return doRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// substitute reserved characters for pcloud +// +// Generally all characters are allowed in filenames, except the NULL +// byte, forward and backslash (/,\ and \0) +func replaceReservedChars(x string) string { + // Backslash for FULLWIDTH REVERSE SOLIDUS + return strings.Replace(x, "\\", "＼", -1) +} + +// restore reserved characters for pcloud +func restoreReservedChars(x string) string { + // FULLWIDTH REVERSE SOLIDUS for Backslash + return strings.Replace(x, "＼", "\\", -1) +} + +// readMetaDataForPath reads the metadata from the path +func (f *Fs) readMetaDataForPath(path string) (info *api.Item, err error) { + // defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err) + leaf, directoryID, err := f.dirCache.FindRootAndPath(path, false) + if err != nil { + if err == fs.ErrorDirNotFound { + return nil, fs.ErrorObjectNotFound + } + return nil, err + } + + found, err := f.listAll(directoryID, false, true, func(item *api.Item) bool { + if item.Name == leaf { + info = item + return true + } + return false + }) + if err != nil { + return nil, err + } + if !found { + return nil, fs.ErrorObjectNotFound + } + return info, nil +} + +// errorHandler parses a non 2xx error response into an error +func errorHandler(resp *http.Response) error { + // Decode error response + errResponse := new(api.Error) + err := rest.DecodeJSON(resp, &errResponse) + if err != nil { + fs.Debugf(nil, "Couldn't decode error response: %v", err) + } + if errResponse.ErrorString == "" { + errResponse.ErrorString = resp.Status + } + if errResponse.Result == 0 { + errResponse.Result = resp.StatusCode + } + return errResponse +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + root = parsePath(root) + oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig) + if err != nil { + log.Fatalf("Failed to configure Pcloud: %v", err) + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + srv: rest.NewClient(oAuthClient).SetRoot(rootURL), + pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), + } + f.features = (&fs.Features{ + CaseInsensitive: false, + CanHaveEmptyDirectories: true, + }).Fill(f) + f.srv.SetErrorHandler(errorHandler) + + // Renew the token in the background + f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { + _, err := f.readMetaDataForPath("") + return err + }) + + // Get rootID + f.dirCache = dircache.New(root, rootID, f) + + // Find the current root + err = f.dirCache.FindRoot(false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + newF := *f + newF.dirCache = dircache.New(newRoot, rootID, &newF) + newF.root = newRoot + // Make new Fs which is the parent + err = newF.dirCache.FindRoot(false) + if err != nil { + // No root so return old f + return f, nil + } + _, err := newF.newObjectWithInfo(remote, nil) + if err != nil { + if err == fs.ErrorObjectNotFound { + // File doesn't exist so return old f + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return &newF, fs.ErrorIsFile + } + return f, nil +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *api.Item) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + // Set info + err = o.setMetaData(info) + } else { + err = o.readMetaData() // reads info and meta, returning an error + } + if err != nil { + return nil, err + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) { + // Find the leaf in pathID + found, err = f.listAll(pathID, true, false, func(item *api.Item) bool { + if item.Name == leaf { + pathIDOut = item.ID + return true + } + return false + }) + return pathIDOut, found, err +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) { + // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) + var resp *http.Response + var result api.ItemResult + opts := rest.Opts{ + Method: "POST", + Path: "/createfolder", + Parameters: url.Values{}, + } + opts.Parameters.Set("name", replaceReservedChars(leaf)) + opts.Parameters.Set("folderid", dirIDtoNumber(pathID)) + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &result) + err = result.Error.Update(err) + return shouldRetry(resp, err) + }) + if err != nil { + //fmt.Printf("...Error %v\n", err) + return "", err + } + // fmt.Printf("...Id %q\n", *info.Id) + return result.Metadata.ID, nil +} + +// Converts a dirID which is usually 'd' followed by digits into just +// the digits +func dirIDtoNumber(dirID string) string { + if len(dirID) > 0 && dirID[0] == 'd' { + return dirID[1:] + } + fs.Debugf(nil, "Invalid directory id %q", dirID) + return dirID +} + +// Converts a fileID which is usually 'f' followed by digits into just +// the digits +func fileIDtoNumber(fileID string) string { + if len(fileID) > 0 && fileID[0] == 'f' { + return fileID[1:] + } + fs.Debugf(nil, "Invalid filee id %q", fileID) + return fileID +} + +// list the objects into the function supplied +// +// If directories is set it only sends directories +// User function to process a File item from listAll +// +// Should return true to finish processing +type listAllFn func(*api.Item) bool + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { + opts := rest.Opts{ + Method: "GET", + Path: "/listfolder", + Parameters: url.Values{}, + } + opts.Parameters.Set("folderid", dirIDtoNumber(dirID)) + // FIXME can do recursive + + var result api.ItemResult + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &result) + err = result.Error.Update(err) + return shouldRetry(resp, err) + }) + if err != nil { + return found, errors.Wrap(err, "couldn't list files") + } + for i := range result.Metadata.Contents { + item := &result.Metadata.Contents[i] + if item.IsFolder { + if filesOnly { + continue + } + } else { + if directoriesOnly { + continue + } + } + item.Name = restoreReservedChars(item.Name) + if fn(item) { + found = true + break + } + } + return +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + err = f.dirCache.FindRoot(false) + if err != nil { + return nil, err + } + directoryID, err := f.dirCache.FindDir(dir, false) + if err != nil { + return nil, err + } + var iErr error + _, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { + remote := path.Join(dir, info.Name) + if info.IsFolder { + // cache the directory ID for later lookups + f.dirCache.Put(remote, info.ID) + d := fs.NewDir(remote, info.ModTime()).SetID(info.ID) + // FIXME more info from dir? + entries = append(entries, d) + } else { + o, err := f.newObjectWithInfo(remote, info) + if err != nil { + iErr = err + return true + } + entries = append(entries, o) + } + return false + }) + if err != nil { + return nil, err + } + if iErr != nil { + return nil, iErr + } + return entries, nil +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Returns the object, leaf, directoryID and error +// +// Used to create new objects +func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { + // Create the directory for the object if it doesn't exist + leaf, directoryID, err = f.dirCache.FindRootAndPath(remote, true) + if err != nil { + return + } + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + } + return o, leaf, directoryID, nil +} + +// Put the object into the container +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime() + + o, _, _, err := f.createObject(remote, modTime, size) + if err != nil { + return nil, err + } + return o, o.Update(in, src, options...) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + err := f.dirCache.FindRoot(true) + if err != nil { + return err + } + if dir != "" { + _, err = f.dirCache.FindDir(dir, true) + } + return err +} + +// purgeCheck removes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(dir string, check bool) error { + root := path.Join(f.root, dir) + if root == "" { + return errors.New("can't purge root directory") + } + dc := f.dirCache + err := dc.FindRoot(false) + if err != nil { + return err + } + rootID, err := dc.FindDir(dir, false) + if err != nil { + return err + } + + opts := rest.Opts{ + Method: "POST", + Path: "/deletefolder", + Parameters: url.Values{}, + } + opts.Parameters.Set("folderid", dirIDtoNumber(rootID)) + if !check { + opts.Path = "/deletefolderrecursive" + } + var resp *http.Response + var result api.ItemResult + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &result) + err = result.Error.Update(err) + return shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "rmdir failed") + } + f.dirCache.FlushDir(dir) + if err != nil { + return err + } + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + return f.purgeCheck(dir, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + err := srcObj.readMetaData() + if err != nil { + return nil, err + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + // Copy the object + opts := rest.Opts{ + Method: "POST", + Path: "/copyfile", + Parameters: url.Values{}, + } + opts.Parameters.Set("fileid", fileIDtoNumber(srcObj.id)) + opts.Parameters.Set("toname", replaceReservedChars(leaf)) + opts.Parameters.Set("tofolderid", dirIDtoNumber(directoryID)) + opts.Parameters.Set("mtime", fmt.Sprintf("%d", srcObj.modTime.Unix())) + var resp *http.Response + var result api.ItemResult + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &result) + err = result.Error.Update(err) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + err = dstObj.setMetaData(&result.Metadata) + if err != nil { + return nil, err + } + return dstObj, nil +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + return f.purgeCheck("", false) +} + +// CleanUp empties the trash +func (f *Fs) CleanUp() error { + err := f.dirCache.FindRoot(false) + if err != nil { + return err + } + opts := rest.Opts{ + Method: "POST", + Path: "/trash_clear", + Parameters: url.Values{}, + } + opts.Parameters.Set("folderid", dirIDtoNumber(f.dirCache.RootID())) + var resp *http.Response + var result api.Error + return f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &result) + err = result.Update(err) + return shouldRetry(resp, err) + }) +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + // Do the move + opts := rest.Opts{ + Method: "POST", + Path: "/renamefile", + Parameters: url.Values{}, + } + opts.Parameters.Set("fileid", fileIDtoNumber(srcObj.id)) + opts.Parameters.Set("toname", replaceReservedChars(leaf)) + opts.Parameters.Set("tofolderid", dirIDtoNumber(directoryID)) + var resp *http.Response + var result api.ItemResult + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &result) + err = result.Error.Update(err) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + + err = dstObj.setMetaData(&result.Metadata) + if err != nil { + return nil, err + } + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := path.Join(srcFs.root, srcRemote) + dstPath := path.Join(f.root, dstRemote) + + // Refuse to move to or from the root + if srcPath == "" || dstPath == "" { + fs.Debugf(src, "DirMove error: Can't move root") + return errors.New("can't move root directory") + } + + // find the root src directory + err := srcFs.dirCache.FindRoot(false) + if err != nil { + return err + } + + // find the root dst directory + if dstRemote != "" { + err = f.dirCache.FindRoot(true) + if err != nil { + return err + } + } else { + if f.dirCache.FoundRoot() { + return fs.ErrorDirExists + } + } + + // Find ID of dst parent, creating subdirs if necessary + var leaf, directoryID string + findPath := dstRemote + if dstRemote == "" { + findPath = f.root + } + leaf, directoryID, err = f.dirCache.FindPath(findPath, true) + if err != nil { + return err + } + + // Check destination does not exist + if dstRemote != "" { + _, err = f.dirCache.FindDir(dstRemote, false) + if err == fs.ErrorDirNotFound { + // OK + } else if err != nil { + return err + } else { + return fs.ErrorDirExists + } + } + + // Find ID of src + srcID, err := srcFs.dirCache.FindDir(srcRemote, false) + if err != nil { + return err + } + + // Do the move + opts := rest.Opts{ + Method: "POST", + Path: "/renamefolder", + Parameters: url.Values{}, + } + opts.Parameters.Set("folderid", dirIDtoNumber(srcID)) + opts.Parameters.Set("toname", replaceReservedChars(leaf)) + opts.Parameters.Set("tofolderid", dirIDtoNumber(directoryID)) + var resp *http.Response + var result api.ItemResult + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &result) + err = result.Error.Update(err) + return shouldRetry(resp, err) + }) + if err != nil { + return err + } + + srcFs.dirCache.FlushDir(srcRemote) + return nil +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *Fs) DirCacheFlush() { + f.dirCache.ResetRoot() +} + +// About gets quota information +func (f *Fs) About() (usage *fs.Usage, err error) { + opts := rest.Opts{ + Method: "POST", + Path: "/userinfo", + } + var resp *http.Response + var q api.UserInfo + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &q) + err = q.Error.Update(err) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "about failed") + } + usage = &fs.Usage{ + Total: fs.NewUsageValue(q.Quota), // quota of bytes that can be used + Used: fs.NewUsageValue(q.UsedQuota), // bytes in use + Free: fs.NewUsageValue(q.Quota - q.UsedQuota), // bytes which can be uploaded before reaching the quota + } + return usage, nil +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5 | hash.SHA1) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// getHashes fetches the hashes into the object +func (o *Object) getHashes() (err error) { + var resp *http.Response + var result api.ChecksumFileResult + opts := rest.Opts{ + Method: "GET", + Path: "/checksumfile", + Parameters: url.Values{}, + } + opts.Parameters.Set("fileid", fileIDtoNumber(o.id)) + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(&opts, nil, &result) + err = result.Error.Update(err) + return shouldRetry(resp, err) + }) + if err != nil { + return err + } + o.setHashes(&result.Hashes) + return o.setMetaData(&result.Metadata) +} + +// Hash returns the SHA-1 of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.MD5 && t != hash.SHA1 { + return "", hash.ErrUnsupported + } + if o.md5 == "" && o.sha1 == "" { + err := o.getHashes() + if err != nil { + return "", errors.Wrap(err, "failed to get hash") + } + } + if t == hash.MD5 { + return o.md5, nil + } + return o.sha1, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return 0 + } + return o.size +} + +// setMetaData sets the metadata from info +func (o *Object) setMetaData(info *api.Item) (err error) { + if info.IsFolder { + return errors.Wrapf(fs.ErrorNotAFile, "%q is a folder", o.remote) + } + o.hasMetaData = true + o.size = info.Size + o.modTime = info.ModTime() + o.id = info.ID + return nil +} + +// setHashes sets the hashes from that passed in +func (o *Object) setHashes(hashes *api.Hashes) { + o.sha1 = hashes.SHA1 + o.md5 = hashes.MD5 +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData() (err error) { + if o.hasMetaData { + return nil + } + info, err := o.fs.readMetaDataForPath(o.remote) + if err != nil { + //if apiErr, ok := err.(*api.Error); ok { + // FIXME + // if apiErr.Code == "not_found" || apiErr.Code == "trashed" { + // return fs.ErrorObjectNotFound + // } + //} + return err + } + return o.setMetaData(info) +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + // Pcloud doesn't have a way of doing this so returning this + // error will cause the file to be re-uploaded to set the time. + return fs.ErrorCantSetModTime +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// downloadURL fetches the download link +func (o *Object) downloadURL() (URL string, err error) { + if o.id == "" { + return "", errors.New("can't download - no id") + } + if o.link.IsValid() { + return o.link.URL(), nil + } + var resp *http.Response + var result api.GetFileLinkResult + opts := rest.Opts{ + Method: "GET", + Path: "/getfilelink", + Parameters: url.Values{}, + } + opts.Parameters.Set("fileid", fileIDtoNumber(o.id)) + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(&opts, nil, &result) + err = result.Error.Update(err) + return shouldRetry(resp, err) + }) + if err != nil { + return "", err + } + if !result.IsValid() { + return "", errors.Errorf("fetched invalid link %+v", result) + } + o.link = &result + return o.link.URL(), nil +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + url, err := o.downloadURL() + if err != nil { + return nil, err + } + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + RootURL: url, + Options: options, + } + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + return resp.Body, err +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// If existing is set then it updates the object rather than creating a new one +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + o.fs.tokenRenewer.Start() + defer o.fs.tokenRenewer.Stop() + + size := src.Size() // NB can upload without size + modTime := src.ModTime() + remote := o.Remote() + + // Create the directory for the object if it doesn't exist + leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(remote, true) + if err != nil { + return err + } + + // Experiments with pcloud indicate that it doesn't like any + // form of request which doesn't have a Content-Length. + // According to the docs if you close the connection at the + // end then it should work without Content-Length, but I + // couldn't get this to work using opts.Close (which sets + // http.Request.Close). + // + // This means that chunked transfer encoding needs to be + // disabled and a Content-Length needs to be supplied. This + // also rules out streaming. + // + // Docs: https://docs.pcloud.com/methods/file/uploadfile.html + var resp *http.Response + var result api.UploadFileResponse + opts := rest.Opts{ + Method: "PUT", + Path: "/uploadfile", + Body: in, + ContentType: fs.MimeType(o), + ContentLength: &size, + Parameters: url.Values{}, + TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding + } + leaf = replaceReservedChars(leaf) + opts.Parameters.Set("filename", leaf) + opts.Parameters.Set("folderid", dirIDtoNumber(directoryID)) + opts.Parameters.Set("nopartial", "1") + opts.Parameters.Set("mtime", fmt.Sprintf("%d", modTime.Unix())) + + // Special treatment for a 0 length upload. This doesn't work + // with PUT even with Content-Length set (by setting + // opts.Body=0), so upload it as a multpart form POST with + // Content-Length set. + if size == 0 { + formReader, contentType, err := rest.MultipartUpload(in, opts.Parameters, "content", leaf) + if err != nil { + return errors.Wrap(err, "failed to make multipart upload for 0 length file") + } + formBody, err := ioutil.ReadAll(formReader) + if err != nil { + return errors.Wrap(err, "failed to read multipart upload for 0 length file") + } + length := int64(len(formBody)) + + opts.ContentType = contentType + opts.Body = bytes.NewBuffer(formBody) + opts.Method = "POST" + opts.Parameters = nil + opts.ContentLength = &length + } + + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(&opts, nil, &result) + err = result.Error.Update(err) + return shouldRetry(resp, err) + }) + if err != nil { + // sometimes pcloud leaves a half complete file on + // error, so delete it if it exists + delObj, delErr := o.fs.NewObject(o.remote) + if delErr == nil && delObj != nil { + _ = delObj.Remove() + } + return err + } + if len(result.Items) != 1 { + return errors.Errorf("failed to upload %v - not sure why", o) + } + o.setHashes(&result.Checksums[0]) + return o.setMetaData(&result.Items[0]) +} + +// Remove an object +func (o *Object) Remove() error { + opts := rest.Opts{ + Method: "POST", + Path: "/deletefile", + Parameters: url.Values{}, + } + var result api.ItemResult + opts.Parameters.Set("fileid", fileIDtoNumber(o.id)) + return o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.CallJSON(&opts, nil, &result) + err = result.Error.Update(err) + return shouldRetry(resp, err) + }) +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.id +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.CleanUpper = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.IDer = (*Object)(nil) +) diff --git a/.rclone_repo/backend/pcloud/pcloud_test.go b/.rclone_repo/backend/pcloud/pcloud_test.go new file mode 100755 index 0000000..243ecb3 --- /dev/null +++ b/.rclone_repo/backend/pcloud/pcloud_test.go @@ -0,0 +1,17 @@ +// Test Pcloud filesystem interface +package pcloud_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/pcloud" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestPcloud:", + NilObject: (*pcloud.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/qingstor/qingstor.go b/.rclone_repo/backend/qingstor/qingstor.go new file mode 100755 index 0000000..bb36274 --- /dev/null +++ b/.rclone_repo/backend/qingstor/qingstor.go @@ -0,0 +1,1008 @@ +// Package qingstor provides an interface to QingStor object storage +// Home: https://www.qingcloud.com/ + +// +build !plan9 + +package qingstor + +import ( + "fmt" + "io" + "net/http" + "path" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/fs/walk" + "github.com/pkg/errors" + qsConfig "github.com/yunify/qingstor-sdk-go/config" + qsErr "github.com/yunify/qingstor-sdk-go/request/errors" + qs "github.com/yunify/qingstor-sdk-go/service" +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "qingstor", + Description: "QingCloud Object Storage", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "env_auth", + Help: "Get QingStor credentials from runtime. Only applies if access_key_id and secret_access_key is blank.", + Default: false, + Examples: []fs.OptionExample{{ + Value: "false", + Help: "Enter QingStor credentials in the next step", + }, { + Value: "true", + Help: "Get QingStor credentials from the environment (env vars or IAM)", + }}, + }, { + Name: "access_key_id", + Help: "QingStor Access Key ID\nLeave blank for anonymous access or runtime credentials.", + }, { + Name: "secret_access_key", + Help: "QingStor Secret Access Key (password)\nLeave blank for anonymous access or runtime credentials.", + }, { + Name: "endpoint", + Help: "Enter a endpoint URL to connection QingStor API.\nLeave blank will use the default value \"https://qingstor.com:443\"", + }, { + Name: "zone", + Help: "Zone to connect to.\nDefault is \"pek3a\".", + Examples: []fs.OptionExample{{ + Value: "pek3a", + Help: "The Beijing (China) Three Zone\nNeeds location constraint pek3a.", + }, { + Value: "sh1a", + Help: "The Shanghai (China) First Zone\nNeeds location constraint sh1a.", + }, { + Value: "gd2a", + Help: "The Guangdong (China) Second Zone\nNeeds location constraint gd2a.", + }}, + }, { + Name: "connection_retries", + Help: "Number of connnection retries.", + Default: 3, + Advanced: true, + }}, + }) +} + +// Constants +const ( + listLimitSize = 1000 // Number of items to read at once + maxSizeForCopy = 1024 * 1024 * 1024 * 5 // The maximum size of object we can COPY +) + +// Globals +func timestampToTime(tp int64) time.Time { + timeLayout := time.RFC3339Nano + ts := time.Unix(tp, 0).Format(timeLayout) + tm, _ := time.Parse(timeLayout, ts) + return tm.UTC() +} + +// Options defines the configuration for this backend +type Options struct { + EnvAuth bool `config:"env_auth"` + AccessKeyID string `config:"access_key_id"` + SecretAccessKey string `config:"secret_access_key"` + Endpoint string `config:"endpoint"` + Zone string `config:"zone"` + ConnectionRetries int `config:"connection_retries"` +} + +// Fs represents a remote qingstor server +type Fs struct { + name string // The name of the remote + root string // The root is a subdir, is a special object + opt Options // parsed options + features *fs.Features // optional features + svc *qs.Service // The connection to the qingstor server + zone string // The zone we are working on + bucket string // The bucket we are working on + bucketOKMu sync.Mutex // mutex to protect bucketOK and bucketDeleted + bucketOK bool // true if we have created the bucket + bucketDeleted bool // true if we have deleted the bucket +} + +// Object describes a qingstor object +type Object struct { + // Will definitely have everything but meta which may be nil + // + // List will read everything but meta & mimeType - to fill + // that in you need to call readMetaData + fs *Fs // what this object is part of + remote string // object of remote + etag string // md5sum of the object + size int64 // length of the object content + mimeType string // ContentType of object - may be "" + lastModified time.Time // Last modified + encrypted bool // whether the object is encryption + algo string // Custom encryption algorithms +} + +// ------------------------------------------------------------ + +// Pattern to match a qingstor path +var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`) + +// parseParse parses a qingstor 'url' +func qsParsePath(path string) (bucket, key string, err error) { + // Pattern to match a qingstor path + parts := matcher.FindStringSubmatch(path) + if parts == nil { + err = errors.Errorf("Couldn't parse bucket out of qingstor path %q", path) + } else { + bucket, key = parts[1], parts[2] + key = strings.Trim(key, "/") + } + return +} + +// Split an URL into three parts: protocol host and port +func qsParseEndpoint(endpoint string) (protocol, host, port string, err error) { + /* + Pattern to match a endpoint, + eg: "http(s)://qingstor.com:443" --> "http(s)", "qingstor.com", 443 + "http(s)//qingstor.com" --> "http(s)", "qingstor.com", "" + "qingstor.com" --> "", "qingstor.com", "" + */ + defer func() { + if r := recover(); r != nil { + switch x := r.(type) { + case error: + err = x + default: + err = nil + } + } + }() + var mather = regexp.MustCompile(`^(?:(http|https)://)*(\w+\.(?:[\w\.])*)(?::(\d{0,5}))*$`) + parts := mather.FindStringSubmatch(endpoint) + protocol, host, port = parts[1], parts[2], parts[3] + return +} + +// qsConnection makes a connection to qingstor +func qsServiceConnection(opt *Options) (*qs.Service, error) { + accessKeyID := opt.AccessKeyID + secretAccessKey := opt.SecretAccessKey + + switch { + case opt.EnvAuth: + // No need for empty checks if "env_auth" is true + case accessKeyID == "" && secretAccessKey == "": + // if no access key/secret and iam is explicitly disabled then fall back to anon interaction + case accessKeyID == "": + return nil, errors.New("access_key_id not found") + case secretAccessKey == "": + return nil, errors.New("secret_access_key not found") + } + + protocol := "https" + host := "qingstor.com" + port := 443 + + endpoint := opt.Endpoint + if endpoint != "" { + _protocol, _host, _port, err := qsParseEndpoint(endpoint) + + if err != nil { + return nil, fmt.Errorf("The endpoint \"%s\" format error", endpoint) + } + + if _protocol != "" { + protocol = _protocol + } + host = _host + if _port != "" { + port, _ = strconv.Atoi(_port) + } else if protocol == "http" { + port = 80 + } + + } + + cf, err := qsConfig.NewDefault() + if err != nil { + return nil, err + } + cf.AccessKeyID = accessKeyID + cf.SecretAccessKey = secretAccessKey + cf.Protocol = protocol + cf.Host = host + cf.Port = port + cf.ConnectionRetries = opt.ConnectionRetries + cf.Connection = fshttp.NewClient(fs.Config) + + return qs.Init(cf) +} + +// NewFs constructs an Fs from the path, bucket:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + bucket, key, err := qsParsePath(root) + if err != nil { + return nil, err + } + svc, err := qsServiceConnection(opt) + if err != nil { + return nil, err + } + + if opt.Zone == "" { + opt.Zone = "pek3a" + } + + f := &Fs{ + name: name, + root: key, + opt: *opt, + svc: svc, + zone: opt.Zone, + bucket: bucket, + } + f.features = (&fs.Features{ + ReadMimeType: true, + WriteMimeType: true, + BucketBased: true, + }).Fill(f) + + if f.root != "" { + if !strings.HasSuffix(f.root, "/") { + f.root += "/" + } + //Check to see if the object exists + bucketInit, err := svc.Bucket(bucket, opt.Zone) + if err != nil { + return nil, err + } + _, err = bucketInit.HeadObject(key, &qs.HeadObjectInput{}) + if err == nil { + f.root = path.Dir(key) + if f.root == "." { + f.root = "" + } else { + f.root += "/" + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + } + return f, nil +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + if f.root == "" { + return f.bucket + } + return f.bucket + "/" + f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + if f.root == "" { + return fmt.Sprintf("QingStor bucket %s", f.bucket) + } + return fmt.Sprintf("QingStor bucket %s root %s", f.bucket, f.root) +} + +// Precision of the remote +func (f *Fs) Precision() time.Duration { + //return time.Nanosecond + //Not supported temporary + return fs.ModTimeNotSupported +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) + //return hash.HashSet(hash.HashNone) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Put created a new object +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + fsObj := &Object{ + fs: f, + remote: src.Remote(), + } + return fsObj, fsObj.Update(in, src, options...) +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + err := f.Mkdir("") + if err != nil { + return nil, err + } + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + srcFs := srcObj.fs + key := f.root + remote + source := path.Join("/"+srcFs.bucket, srcFs.root+srcObj.remote) + + fs.Debugf(f, "Copied, source key is: %s, and dst key is: %s", source, key) + req := qs.PutObjectInput{ + XQSCopySource: &source, + } + bucketInit, err := f.svc.Bucket(f.bucket, f.zone) + + if err != nil { + return nil, err + } + _, err = bucketInit.PutObject(key, &req) + if err != nil { + fs.Debugf(f, "Copied Faild, API Error: %v", err) + return nil, err + } + return f.NewObject(remote) +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// Return an Object from a path +// +//If it can't be found it returns the error ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *qs.KeyType) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + if info != nil { + // Set info + if info.Size != nil { + o.size = *info.Size + } + + if info.Etag != nil { + o.etag = qs.StringValue(info.Etag) + } + if info.Modified == nil { + fs.Logf(o, "Failed to read last modified") + o.lastModified = time.Now() + } else { + o.lastModified = timestampToTime(int64(*info.Modified)) + } + + if info.MimeType != nil { + o.mimeType = qs.StringValue(info.MimeType) + } + + if info.Encrypted != nil { + o.encrypted = qs.BoolValue(info.Encrypted) + } + + } else { + err := o.readMetaData() // reads info and meta, returning an error + if err != nil { + return nil, err + } + } + return o, nil +} + +// listFn is called from list to handle an object. +type listFn func(remote string, object *qs.KeyType, isDirectory bool) error + +// list the objects into the function supplied +// +// dir is the starting directory, "" for root +// +// Set recurse to read sub directories +func (f *Fs) list(dir string, recurse bool, fn listFn) error { + prefix := f.root + if dir != "" { + prefix += dir + "/" + } + + delimiter := "" + if !recurse { + delimiter = "/" + } + + maxLimit := int(listLimitSize) + var marker *string + + for { + bucketInit, err := f.svc.Bucket(f.bucket, f.zone) + if err != nil { + return err + } + // FIXME need to implement ALL loop + req := qs.ListObjectsInput{ + Delimiter: &delimiter, + Prefix: &prefix, + Limit: &maxLimit, + Marker: marker, + } + resp, err := bucketInit.ListObjects(&req) + if err != nil { + if e, ok := err.(*qsErr.QingStorError); ok { + if e.StatusCode == http.StatusNotFound { + err = fs.ErrorDirNotFound + } + } + return err + } + rootLength := len(f.root) + if !recurse { + for _, commonPrefix := range resp.CommonPrefixes { + if commonPrefix == nil { + fs.Logf(f, "Nil common prefix received") + continue + } + remote := *commonPrefix + if !strings.HasPrefix(remote, f.root) { + fs.Logf(f, "Odd name received %q", remote) + continue + } + remote = remote[rootLength:] + if strings.HasSuffix(remote, "/") { + remote = remote[:len(remote)-1] + } + + err = fn(remote, &qs.KeyType{Key: &remote}, true) + if err != nil { + return err + } + } + } + + for _, object := range resp.Keys { + key := qs.StringValue(object.Key) + if !strings.HasPrefix(key, f.root) { + fs.Logf(f, "Odd name received %q", key) + continue + } + remote := key[rootLength:] + err = fn(remote, object, false) + if err != nil { + return err + } + } + // Use NextMarker if set, otherwise use last Key + if resp.NextMarker == nil || *resp.NextMarker == "" { + //marker = resp.Keys[len(resp.Keys)-1].Key + break + } else { + marker = resp.NextMarker + } + } + return nil +} + +// Convert a list item into a BasicInfo +func (f *Fs) itemToDirEntry(remote string, object *qs.KeyType, isDirectory bool) (fs.DirEntry, error) { + if isDirectory { + size := int64(0) + if object.Size != nil { + size = *object.Size + } + d := fs.NewDir(remote, time.Time{}).SetSize(size) + return d, nil + } + o, err := f.newObjectWithInfo(remote, object) + if err != nil { + return nil, err + } + return o, nil +} + +// mark the bucket as being OK +func (f *Fs) markBucketOK() { + if f.bucket != "" { + f.bucketOKMu.Lock() + f.bucketOK = true + f.bucketDeleted = false + f.bucketOKMu.Unlock() + } +} + +// listDir lists files and directories to out +func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { + // List the objects and directories + err = f.list(dir, false, func(remote string, object *qs.KeyType, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + if entry != nil { + entries = append(entries, entry) + } + return nil + }) + if err != nil { + return nil, err + } + // bucket must be present if listing succeeded + f.markBucketOK() + return entries, nil +} + +// listBuckets lists the buckets to out +func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) { + if dir != "" { + return nil, fs.ErrorListBucketRequired + } + + req := qs.ListBucketsInput{ + Location: &f.zone, + } + resp, err := f.svc.ListBuckets(&req) + if err != nil { + return nil, err + } + + for _, bucket := range resp.Buckets { + d := fs.NewDir(qs.StringValue(bucket.Name), qs.TimeValue(bucket.Created)) + entries = append(entries, d) + } + return entries, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + if f.bucket == "" { + return f.listBuckets(dir) + } + return f.listDir(dir) +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + if f.bucket == "" { + return fs.ErrorListBucketRequired + } + list := walk.NewListRHelper(callback) + err = f.list(dir, true, func(remote string, object *qs.KeyType, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + return list.Add(entry) + }) + if err != nil { + return err + } + // bucket must be present if listing succeeded + f.markBucketOK() + return list.Flush() +} + +// Check if the bucket exists +func (f *Fs) dirExists() (bool, error) { + bucketInit, err := f.svc.Bucket(f.bucket, f.zone) + if err != nil { + return false, err + } + + _, err = bucketInit.Head() + if err == nil { + return true, nil + } + + if e, ok := err.(*qsErr.QingStorError); ok { + if e.StatusCode == http.StatusNotFound { + err = nil + } + } + return false, err +} + +// Mkdir creates the bucket if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + f.bucketOKMu.Lock() + defer f.bucketOKMu.Unlock() + if f.bucketOK { + return nil + } + + bucketInit, err := f.svc.Bucket(f.bucket, f.zone) + if err != nil { + return err + } + /* When delete a bucket, qingstor need about 60 second to sync status; + So, need wait for it sync end if we try to operation a just deleted bucket + */ + retries := 0 + for retries <= 120 { + statistics, err := bucketInit.GetStatistics() + if statistics == nil || err != nil { + break + } + switch *statistics.Status { + case "deleted": + fs.Debugf(f, "Wiat for qingstor sync bucket status, retries: %d", retries) + time.Sleep(time.Second * 1) + retries++ + continue + default: + break + } + break + } + + if !f.bucketDeleted { + exists, err := f.dirExists() + if err == nil { + f.bucketOK = exists + } + if err != nil || exists { + return err + } + } + + _, err = bucketInit.Put() + if e, ok := err.(*qsErr.QingStorError); ok { + if e.StatusCode == http.StatusConflict { + err = nil + } + } + + if err == nil { + f.bucketOK = true + f.bucketDeleted = false + } + + return err +} + +// dirIsEmpty check if the bucket empty +func (f *Fs) dirIsEmpty() (bool, error) { + bucketInit, err := f.svc.Bucket(f.bucket, f.zone) + if err != nil { + return true, err + } + + statistics, err := bucketInit.GetStatistics() + if err != nil { + return true, err + } + + if *statistics.Count == 0 { + return true, nil + } + return false, nil +} + +// Rmdir delete a bucket +func (f *Fs) Rmdir(dir string) error { + f.bucketOKMu.Lock() + defer f.bucketOKMu.Unlock() + if f.root != "" || dir != "" { + return nil + } + + isEmpty, err := f.dirIsEmpty() + if err != nil { + return err + } + if !isEmpty { + fs.Debugf(f, "The bucket %s you tried to delete not empty.", f.bucket) + return errors.New("BucketNotEmpty: The bucket you tried to delete is not empty") + } + + fs.Debugf(f, "Tried to delete the bucket %s", f.bucket) + bucketInit, err := f.svc.Bucket(f.bucket, f.zone) + if err != nil { + return err + } + retries := 0 + for retries <= 10 { + _, delErr := bucketInit.Delete() + if delErr != nil { + if e, ok := delErr.(*qsErr.QingStorError); ok { + switch e.Code { + // The status of "lease" takes a few seconds to "ready" when creating a new bucket + // wait for lease status ready + case "lease_not_ready": + fs.Debugf(f, "QingStor bucket lease not ready, retries: %d", retries) + retries++ + time.Sleep(time.Second * 1) + continue + default: + err = e + break + } + } + } else { + err = delErr + } + break + } + + if err == nil { + f.bucketOK = false + f.bucketDeleted = true + } + return err +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData() (err error) { + bucketInit, err := o.fs.svc.Bucket(o.fs.bucket, o.fs.zone) + if err != nil { + return err + } + + key := o.fs.root + o.remote + fs.Debugf(o, "Read metadata of key: %s", key) + resp, err := bucketInit.HeadObject(key, &qs.HeadObjectInput{}) + if err != nil { + fs.Debugf(o, "Read metadata faild, API Error: %v", err) + if e, ok := err.(*qsErr.QingStorError); ok { + if e.StatusCode == http.StatusNotFound { + return fs.ErrorObjectNotFound + } + } + return err + } + // Ignore missing Content-Length assuming it is 0 + if resp.ContentLength != nil { + o.size = *resp.ContentLength + } + + if resp.ETag != nil { + o.etag = qs.StringValue(resp.ETag) + } + + if resp.LastModified == nil { + fs.Logf(o, "Failed to read last modified from HEAD: %v", err) + o.lastModified = time.Now() + } else { + o.lastModified = *resp.LastModified + } + + if resp.ContentType != nil { + o.mimeType = qs.StringValue(resp.ContentType) + } + + if resp.XQSEncryptionCustomerAlgorithm != nil { + o.algo = qs.StringValue(resp.XQSEncryptionCustomerAlgorithm) + o.encrypted = true + } + + return nil +} + +// ModTime returns the modification date of the file +// It should return a best guess if one isn't available +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata, %v", err) + return time.Now() + } + modTime := o.lastModified + return modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + err := o.readMetaData() + if err != nil { + return err + } + o.lastModified = modTime + mimeType := fs.MimeType(o) + + if o.size >= maxSizeForCopy { + fs.Debugf(o, "SetModTime is unsupported for objects bigger than %v bytes", fs.SizeSuffix(maxSizeForCopy)) + return nil + } + // Copy the object to itself to update the metadata + key := o.fs.root + o.remote + sourceKey := path.Join("/", o.fs.bucket, key) + + bucketInit, err := o.fs.svc.Bucket(o.fs.bucket, o.fs.zone) + if err != nil { + return err + } + + req := qs.PutObjectInput{ + XQSCopySource: &sourceKey, + ContentType: &mimeType, + } + _, err = bucketInit.PutObject(key, &req) + + return err +} + +// Open opens the file for read. Call Close() on the returned io.ReadCloser +func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) { + bucketInit, err := o.fs.svc.Bucket(o.fs.bucket, o.fs.zone) + if err != nil { + return nil, err + } + + key := o.fs.root + o.remote + req := qs.GetObjectInput{} + for _, option := range options { + switch option.(type) { + case *fs.RangeOption, *fs.SeekOption: + _, value := option.Header() + req.Range = &value + default: + if option.Mandatory() { + fs.Logf(o, "Unsupported mandatory option: %v", option) + } + } + } + resp, err := bucketInit.GetObject(key, &req) + if err != nil { + return nil, err + } + return resp.Body, nil +} + +// Update in to the object +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + // The maximum size of upload object is multipartUploadSize * MaxMultipleParts + err := o.fs.Mkdir("") + if err != nil { + return err + } + + key := o.fs.root + o.remote + // Guess the content type + mimeType := fs.MimeType(src) + + req := uploadInput{ + body: in, + qsSvc: o.fs.svc, + bucket: o.fs.bucket, + zone: o.fs.zone, + key: key, + mimeType: mimeType, + } + uploader := newUploader(&req) + + err = uploader.upload() + if err != nil { + return err + } + // Read Metadata of object + err = o.readMetaData() + return err +} + +// Remove this object +func (o *Object) Remove() error { + bucketInit, err := o.fs.svc.Bucket(o.fs.bucket, o.fs.zone) + if err != nil { + return err + } + + key := o.fs.root + o.remote + _, err = bucketInit.DeleteObject(key) + return err +} + +// Fs returns read only access to the Fs that this object is part of +func (o *Object) Fs() fs.Info { + return o.fs +} + +var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`) + +// Hash returns the selected checksum of the file +// If no checksum is available it returns "" +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.MD5 { + return "", hash.ErrUnsupported + } + etag := strings.Trim(strings.ToLower(o.etag), `"`) + // Check the etag is a valid md5sum + if !matchMd5.MatchString(etag) { + fs.Debugf(o, "Invalid md5sum (probably multipart uploaded) - ignoring: %q", etag) + return "", nil + } + return etag, nil +} + +// Storable says whether this object can be stored +func (o *Object) Storable() bool { + return true +} + +// String returns a description of the Object +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Size returns the size of the file +func (o *Object) Size() int64 { + return o.size +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return "" + } + return o.mimeType +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.Copier = &Fs{} + _ fs.Object = &Object{} + _ fs.ListRer = &Fs{} + _ fs.MimeTyper = &Object{} +) diff --git a/.rclone_repo/backend/qingstor/qingstor_test.go b/.rclone_repo/backend/qingstor/qingstor_test.go new file mode 100755 index 0000000..585df51 --- /dev/null +++ b/.rclone_repo/backend/qingstor/qingstor_test.go @@ -0,0 +1,20 @@ +// Test QingStor filesystem interface + +// +build !plan9 + +package qingstor_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/qingstor" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestQingStor:", + NilObject: (*qingstor.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/qingstor/qingstor_unsupported.go b/.rclone_repo/backend/qingstor/qingstor_unsupported.go new file mode 100755 index 0000000..a298023 --- /dev/null +++ b/.rclone_repo/backend/qingstor/qingstor_unsupported.go @@ -0,0 +1,6 @@ +// Build for unsupported platforms to stop go complaining +// about "no buildable Go source files " + +// +build plan9 + +package qingstor diff --git a/.rclone_repo/backend/qingstor/upload.go b/.rclone_repo/backend/qingstor/upload.go new file mode 100755 index 0000000..0640524 --- /dev/null +++ b/.rclone_repo/backend/qingstor/upload.go @@ -0,0 +1,415 @@ +// Upload object to QingStor + +// +build !plan9 + +package qingstor + +import ( + "bytes" + "crypto/md5" + "fmt" + "hash" + "io" + "sort" + "sync" + + "github.com/ncw/rclone/fs" + "github.com/pkg/errors" + qs "github.com/yunify/qingstor-sdk-go/service" +) + +const ( + // maxSinglePartSize = 1024 * 1024 * 1024 * 5 // The maximum allowed size when uploading a single object to QingStor + // maxMultiPartSize = 1024 * 1024 * 1024 * 1 // The maximum allowed part size when uploading a part to QingStor + minMultiPartSize = 1024 * 1024 * 4 // The minimum allowed part size when uploading a part to QingStor + maxMultiParts = 10000 // The maximum allowed number of parts in an multi-part upload +) + +const ( + defaultUploadPartSize = 1024 * 1024 * 64 // The default part size to buffer chunks of a payload into. + defaultUploadConcurrency = 4 // the default number of goroutines to spin up when using multiPartUpload. +) + +func readFillBuf(r io.Reader, b []byte) (offset int, err error) { + for offset < len(b) && err == nil { + var n int + n, err = r.Read(b[offset:]) + offset += n + } + + return offset, err +} + +// uploadInput contains all input for upload requests to QingStor. +type uploadInput struct { + body io.Reader + qsSvc *qs.Service + mimeType string + zone string + bucket string + key string + partSize int64 + concurrency int + maxUploadParts int +} + +// uploader internal structure to manage an upload to QingStor. +type uploader struct { + cfg *uploadInput + totalSize int64 // set to -1 if the size is not known + readerPos int64 // current reader position + readerSize int64 // current reader content size +} + +// newUploader creates a new Uploader instance to upload objects to QingStor. +func newUploader(in *uploadInput) *uploader { + u := &uploader{ + cfg: in, + } + return u +} + +// bucketInit initiate as bucket controller +func (u *uploader) bucketInit() (*qs.Bucket, error) { + bucketInit, err := u.cfg.qsSvc.Bucket(u.cfg.bucket, u.cfg.zone) + return bucketInit, err +} + +// String converts uploader to a string +func (u *uploader) String() string { + return fmt.Sprintf("QingStor bucket %s key %s", u.cfg.bucket, u.cfg.key) +} + +// nextReader returns a seekable reader representing the next packet of data. +// This operation increases the shared u.readerPos counter, but note that it +// does not need to be wrapped in a mutex because nextReader is only called +// from the main thread. +func (u *uploader) nextReader() (io.ReadSeeker, int, error) { + type readerAtSeeker interface { + io.ReaderAt + io.ReadSeeker + } + switch r := u.cfg.body.(type) { + case readerAtSeeker: + var err error + n := u.cfg.partSize + if u.totalSize >= 0 { + bytesLeft := u.totalSize - u.readerPos + + if bytesLeft <= u.cfg.partSize { + err = io.EOF + n = bytesLeft + } + } + reader := io.NewSectionReader(r, u.readerPos, n) + u.readerPos += n + u.readerSize = n + return reader, int(n), err + + default: + part := make([]byte, u.cfg.partSize) + n, err := readFillBuf(r, part) + u.readerPos += int64(n) + u.readerSize = int64(n) + return bytes.NewReader(part[0:n]), n, err + } +} + +// init will initialize all default options. +func (u *uploader) init() { + if u.cfg.concurrency == 0 { + u.cfg.concurrency = defaultUploadConcurrency + } + if u.cfg.partSize == 0 { + u.cfg.partSize = defaultUploadPartSize + } + if u.cfg.maxUploadParts == 0 { + u.cfg.maxUploadParts = maxMultiParts + } + // Try to get the total size for some optimizations + u.totalSize = -1 + switch r := u.cfg.body.(type) { + case io.Seeker: + pos, _ := r.Seek(0, io.SeekCurrent) + defer func() { + _, _ = r.Seek(pos, io.SeekStart) + }() + + n, err := r.Seek(0, io.SeekEnd) + if err != nil { + return + } + u.totalSize = n + + // Try to adjust partSize if it is too small and account for + // integer division truncation. + if u.totalSize/u.cfg.partSize >= int64(u.cfg.partSize) { + // Add one to the part size to account for remainders + // during the size calculation. e.g odd number of bytes. + u.cfg.partSize = (u.totalSize / int64(u.cfg.maxUploadParts)) + 1 + } + } +} + +// singlePartUpload upload a single object that contentLength less than "defaultUploadPartSize" +func (u *uploader) singlePartUpload(buf io.ReadSeeker) error { + bucketInit, _ := u.bucketInit() + + req := qs.PutObjectInput{ + ContentLength: &u.readerPos, + ContentType: &u.cfg.mimeType, + Body: buf, + } + + _, err := bucketInit.PutObject(u.cfg.key, &req) + if err == nil { + fs.Debugf(u, "Upload single objcet finished") + } + return err +} + +// Upload upload a object into QingStor +func (u *uploader) upload() error { + u.init() + + if u.cfg.partSize < minMultiPartSize { + return errors.Errorf("part size must be at least %d bytes", minMultiPartSize) + } + + // Do one read to determine if we have more than one part + reader, _, err := u.nextReader() + if err == io.EOF { // single part + fs.Debugf(u, "Tried to upload a singile object to QingStor") + return u.singlePartUpload(reader) + } else if err != nil { + return errors.Errorf("read upload data failed: %s", err) + } + + fs.Debugf(u, "Treied to upload a multi-part object to QingStor") + mu := multiUploader{uploader: u} + return mu.multiPartUpload(reader) +} + +// internal structure to manage a specific multipart upload to QingStor. +type multiUploader struct { + *uploader + wg sync.WaitGroup + mtx sync.Mutex + err error + uploadID *string + objectParts completedParts + hashMd5 hash.Hash +} + +// keeps track of a single chunk of data being sent to QingStor. +type chunk struct { + buffer io.ReadSeeker + partNumber int + size int64 +} + +// completedParts is a wrapper to make parts sortable by their part number, +// since QingStor required this list to be sent in sorted order. +type completedParts []*qs.ObjectPartType + +func (a completedParts) Len() int { return len(a) } +func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a completedParts) Less(i, j int) bool { return *a[i].PartNumber < *a[j].PartNumber } + +// String converts multiUploader to a string +func (mu *multiUploader) String() string { + if uploadID := mu.uploadID; uploadID != nil { + return fmt.Sprintf("QingStor bucket %s key %s uploadID %s", mu.cfg.bucket, mu.cfg.key, *uploadID) + } + return fmt.Sprintf("QingStor bucket %s key %s uploadID ", mu.cfg.bucket, mu.cfg.key) +} + +// getErr is a thread-safe getter for the error object +func (mu *multiUploader) getErr() error { + mu.mtx.Lock() + defer mu.mtx.Unlock() + return mu.err +} + +// setErr is a thread-safe setter for the error object +func (mu *multiUploader) setErr(e error) { + mu.mtx.Lock() + defer mu.mtx.Unlock() + mu.err = e +} + +// readChunk runs in worker goroutines to pull chunks off of the ch channel +// and send() them as UploadPart requests. +func (mu *multiUploader) readChunk(ch chan chunk) { + defer mu.wg.Done() + for { + c, ok := <-ch + if !ok { + break + } + if mu.getErr() == nil { + if err := mu.send(c); err != nil { + mu.setErr(err) + } + } + } +} + +// initiate init an Multiple Object and obtain UploadID +func (mu *multiUploader) initiate() error { + bucketInit, _ := mu.bucketInit() + req := qs.InitiateMultipartUploadInput{ + ContentType: &mu.cfg.mimeType, + } + fs.Debugf(mu, "Tried to initiate a multi-part upload") + rsp, err := bucketInit.InitiateMultipartUpload(mu.cfg.key, &req) + if err == nil { + mu.uploadID = rsp.UploadID + mu.hashMd5 = md5.New() + } + return err +} + +// send upload a part into QingStor +func (mu *multiUploader) send(c chunk) error { + bucketInit, _ := mu.bucketInit() + req := qs.UploadMultipartInput{ + PartNumber: &c.partNumber, + UploadID: mu.uploadID, + ContentLength: &c.size, + Body: c.buffer, + } + fs.Debugf(mu, "Tried to upload a part to QingStor that partNumber %d and partSize %d", c.partNumber, c.size) + _, err := bucketInit.UploadMultipart(mu.cfg.key, &req) + if err != nil { + return err + } + fs.Debugf(mu, "Upload part finished that partNumber %d and partSize %d", c.partNumber, c.size) + + mu.mtx.Lock() + defer mu.mtx.Unlock() + + _, _ = c.buffer.Seek(0, 0) + _, _ = io.Copy(mu.hashMd5, c.buffer) + + parts := qs.ObjectPartType{PartNumber: &c.partNumber, Size: &c.size} + mu.objectParts = append(mu.objectParts, &parts) + return err +} + +// list list the ObjectParts of an multipart upload +func (mu *multiUploader) list() error { + bucketInit, _ := mu.bucketInit() + + req := qs.ListMultipartInput{ + UploadID: mu.uploadID, + } + fs.Debugf(mu, "Tried to list a multi-part") + rsp, err := bucketInit.ListMultipart(mu.cfg.key, &req) + if err == nil { + mu.objectParts = rsp.ObjectParts + } + return err +} + +// complete complete an multipart upload +func (mu *multiUploader) complete() error { + var err error + if err = mu.getErr(); err != nil { + return err + } + bucketInit, _ := mu.bucketInit() + //if err = mu.list(); err != nil { + // return err + //} + //md5String := fmt.Sprintf("\"%s\"", hex.EncodeToString(mu.hashMd5.Sum(nil))) + + md5String := fmt.Sprintf("\"%x\"", mu.hashMd5.Sum(nil)) + sort.Sort(mu.objectParts) + req := qs.CompleteMultipartUploadInput{ + UploadID: mu.uploadID, + ObjectParts: mu.objectParts, + ETag: &md5String, + } + fs.Debugf(mu, "Tried to complete a multi-part") + _, err = bucketInit.CompleteMultipartUpload(mu.cfg.key, &req) + if err == nil { + fs.Debugf(mu, "Complete multi-part finished") + } + return err +} + +// abort abort an multipart upload +func (mu *multiUploader) abort() error { + var err error + bucketInit, _ := mu.bucketInit() + + if uploadID := mu.uploadID; uploadID != nil { + req := qs.AbortMultipartUploadInput{ + UploadID: uploadID, + } + fs.Debugf(mu, "Tried to abort a multi-part") + _, err = bucketInit.AbortMultipartUpload(mu.cfg.key, &req) + } + + return err +} + +// multiPartUpload upload a multiple object into QingStor +func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) error { + var err error + //Initiate an multi-part upload + if err = mu.initiate(); err != nil { + return err + } + + ch := make(chan chunk, mu.cfg.concurrency) + for i := 0; i < mu.cfg.concurrency; i++ { + mu.wg.Add(1) + go mu.readChunk(ch) + } + + var partNumber int + ch <- chunk{partNumber: partNumber, buffer: firstBuf, size: mu.readerSize} + + for mu.getErr() == nil { + partNumber++ + // This upload exceeded maximum number of supported parts, error now. + if partNumber > mu.cfg.maxUploadParts || partNumber > maxMultiParts { + var msg string + if partNumber > mu.cfg.maxUploadParts { + msg = fmt.Sprintf("exceeded total allowed configured maxUploadParts (%d). "+ + "Adjust PartSize to fit in this limit", mu.cfg.maxUploadParts) + } else { + msg = fmt.Sprintf("exceeded total allowed QingStor limit maxUploadParts (%d). "+ + "Adjust PartSize to fit in this limit", maxMultiParts) + } + mu.setErr(errors.New(msg)) + break + } + + var reader io.ReadSeeker + var nextChunkLen int + reader, nextChunkLen, err = mu.nextReader() + if err != nil && err != io.EOF { + return err + } + if nextChunkLen == 0 && partNumber > 0 { + // No need to upload empty part, if file was empty to start + // with empty single part would of been created and never + // started multipart upload. + break + } + num := partNumber + ch <- chunk{partNumber: num, buffer: reader, size: mu.readerSize} + } + // Wait for all goroutines finish + close(ch) + mu.wg.Wait() + // Complete Multipart Upload + err = mu.complete() + if mu.getErr() != nil || err != nil { + _ = mu.abort() + } + return err +} diff --git a/.rclone_repo/backend/s3/s3.go b/.rclone_repo/backend/s3/s3.go new file mode 100755 index 0000000..2679f71 --- /dev/null +++ b/.rclone_repo/backend/s3/s3.go @@ -0,0 +1,1493 @@ +// Package s3 provides an interface to Amazon S3 oject storage +package s3 + +// FIXME need to prevent anything but ListDir working for s3:// + +/* +Progress of port to aws-sdk + + * Don't really need o.meta at all? + +What happens if you CTRL-C a multipart upload + * get an incomplete upload + * disappears when you delete the bucket +*/ + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net/http" + "path" + "regexp" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/corehandlers" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" + "github.com/aws/aws-sdk-go/aws/defaults" + "github.com/aws/aws-sdk-go/aws/ec2metadata" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/fs/walk" + "github.com/ncw/rclone/lib/rest" + "github.com/ncw/swift" + "github.com/pkg/errors" +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "s3", + Description: "Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)", + NewFs: NewFs, + Options: []fs.Option{{ + Name: fs.ConfigProvider, + Help: "Choose your S3 provider.", + Examples: []fs.OptionExample{{ + Value: "AWS", + Help: "Amazon Web Services (AWS) S3", + }, { + Value: "Ceph", + Help: "Ceph Object Storage", + }, { + Value: "DigitalOcean", + Help: "Digital Ocean Spaces", + }, { + Value: "Dreamhost", + Help: "Dreamhost DreamObjects", + }, { + Value: "IBMCOS", + Help: "IBM COS S3", + }, { + Value: "Minio", + Help: "Minio Object Storage", + }, { + Value: "Wasabi", + Help: "Wasabi Object Storage", + }, { + Value: "Other", + Help: "Any other S3 compatible provider", + }}, + }, { + Name: "env_auth", + Help: "Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).\nOnly applies if access_key_id and secret_access_key is blank.", + Default: false, + Examples: []fs.OptionExample{{ + Value: "false", + Help: "Enter AWS credentials in the next step", + }, { + Value: "true", + Help: "Get AWS credentials from the environment (env vars or IAM)", + }}, + }, { + Name: "access_key_id", + Help: "AWS Access Key ID.\nLeave blank for anonymous access or runtime credentials.", + }, { + Name: "secret_access_key", + Help: "AWS Secret Access Key (password)\nLeave blank for anonymous access or runtime credentials.", + }, { + Name: "region", + Help: "Region to connect to.", + Provider: "AWS", + Examples: []fs.OptionExample{{ + Value: "us-east-1", + Help: "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia or Pacific Northwest.\nLeave location constraint empty.", + }, { + Value: "us-east-2", + Help: "US East (Ohio) Region\nNeeds location constraint us-east-2.", + }, { + Value: "us-west-2", + Help: "US West (Oregon) Region\nNeeds location constraint us-west-2.", + }, { + Value: "us-west-1", + Help: "US West (Northern California) Region\nNeeds location constraint us-west-1.", + }, { + Value: "ca-central-1", + Help: "Canada (Central) Region\nNeeds location constraint ca-central-1.", + }, { + Value: "eu-west-1", + Help: "EU (Ireland) Region\nNeeds location constraint EU or eu-west-1.", + }, { + Value: "eu-west-2", + Help: "EU (London) Region\nNeeds location constraint eu-west-2.", + }, { + Value: "eu-central-1", + Help: "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.", + }, { + Value: "ap-southeast-1", + Help: "Asia Pacific (Singapore) Region\nNeeds location constraint ap-southeast-1.", + }, { + Value: "ap-southeast-2", + Help: "Asia Pacific (Sydney) Region\nNeeds location constraint ap-southeast-2.", + }, { + Value: "ap-northeast-1", + Help: "Asia Pacific (Tokyo) Region\nNeeds location constraint ap-northeast-1.", + }, { + Value: "ap-northeast-2", + Help: "Asia Pacific (Seoul)\nNeeds location constraint ap-northeast-2.", + }, { + Value: "ap-south-1", + Help: "Asia Pacific (Mumbai)\nNeeds location constraint ap-south-1.", + }, { + Value: "sa-east-1", + Help: "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.", + }}, + }, { + Name: "region", + Help: "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.", + Provider: "!AWS", + Examples: []fs.OptionExample{{ + Value: "", + Help: "Use this if unsure. Will use v4 signatures and an empty region.", + }, { + Value: "other-v2-signature", + Help: "Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.", + }}, + }, { + Name: "endpoint", + Help: "Endpoint for S3 API.\nLeave blank if using AWS to use the default endpoint for the region.", + Provider: "AWS", + }, { + Name: "endpoint", + Help: "Endpoint for IBM COS S3 API.\nSpecify if using an IBM COS On Premise.", + Provider: "IBMCOS", + Examples: []fs.OptionExample{{ + Value: "s3-api.us-geo.objectstorage.softlayer.net", + Help: "US Cross Region Endpoint", + }, { + Value: "s3-api.dal.us-geo.objectstorage.softlayer.net", + Help: "US Cross Region Dallas Endpoint", + }, { + Value: "s3-api.wdc-us-geo.objectstorage.softlayer.net", + Help: "US Cross Region Washington DC Endpoint", + }, { + Value: "s3-api.sjc-us-geo.objectstorage.softlayer.net", + Help: "US Cross Region San Jose Endpoint", + }, { + Value: "s3-api.us-geo.objectstorage.service.networklayer.com", + Help: "US Cross Region Private Endpoint", + }, { + Value: "s3-api.dal-us-geo.objectstorage.service.networklayer.com", + Help: "US Cross Region Dallas Private Endpoint", + }, { + Value: "s3-api.wdc-us-geo.objectstorage.service.networklayer.com", + Help: "US Cross Region Washington DC Private Endpoint", + }, { + Value: "s3-api.sjc-us-geo.objectstorage.service.networklayer.com", + Help: "US Cross Region San Jose Private Endpoint", + }, { + Value: "s3.us-east.objectstorage.softlayer.net", + Help: "US Region East Endpoint", + }, { + Value: "s3.us-east.objectstorage.service.networklayer.com", + Help: "US Region East Private Endpoint", + }, { + Value: "s3.us-south.objectstorage.softlayer.net", + Help: "US Region South Endpoint", + }, { + Value: "s3.us-south.objectstorage.service.networklayer.com", + Help: "US Region South Private Endpoint", + }, { + Value: "s3.eu-geo.objectstorage.softlayer.net", + Help: "EU Cross Region Endpoint", + }, { + Value: "s3.fra-eu-geo.objectstorage.softlayer.net", + Help: "EU Cross Region Frankfurt Endpoint", + }, { + Value: "s3.mil-eu-geo.objectstorage.softlayer.net", + Help: "EU Cross Region Milan Endpoint", + }, { + Value: "s3.ams-eu-geo.objectstorage.softlayer.net", + Help: "EU Cross Region Amsterdam Endpoint", + }, { + Value: "s3.eu-geo.objectstorage.service.networklayer.com", + Help: "EU Cross Region Private Endpoint", + }, { + Value: "s3.fra-eu-geo.objectstorage.service.networklayer.com", + Help: "EU Cross Region Frankfurt Private Endpoint", + }, { + Value: "s3.mil-eu-geo.objectstorage.service.networklayer.com", + Help: "EU Cross Region Milan Private Endpoint", + }, { + Value: "s3.ams-eu-geo.objectstorage.service.networklayer.com", + Help: "EU Cross Region Amsterdam Private Endpoint", + }, { + Value: "s3.eu-gb.objectstorage.softlayer.net", + Help: "Great Britan Endpoint", + }, { + Value: "s3.eu-gb.objectstorage.service.networklayer.com", + Help: "Great Britan Private Endpoint", + }, { + Value: "s3.ap-geo.objectstorage.softlayer.net", + Help: "APAC Cross Regional Endpoint", + }, { + Value: "s3.tok-ap-geo.objectstorage.softlayer.net", + Help: "APAC Cross Regional Tokyo Endpoint", + }, { + Value: "s3.hkg-ap-geo.objectstorage.softlayer.net", + Help: "APAC Cross Regional HongKong Endpoint", + }, { + Value: "s3.seo-ap-geo.objectstorage.softlayer.net", + Help: "APAC Cross Regional Seoul Endpoint", + }, { + Value: "s3.ap-geo.objectstorage.service.networklayer.com", + Help: "APAC Cross Regional Private Endpoint", + }, { + Value: "s3.tok-ap-geo.objectstorage.service.networklayer.com", + Help: "APAC Cross Regional Tokyo Private Endpoint", + }, { + Value: "s3.hkg-ap-geo.objectstorage.service.networklayer.com", + Help: "APAC Cross Regional HongKong Private Endpoint", + }, { + Value: "s3.seo-ap-geo.objectstorage.service.networklayer.com", + Help: "APAC Cross Regional Seoul Private Endpoint", + }, { + Value: "s3.mel01.objectstorage.softlayer.net", + Help: "Melbourne Single Site Endpoint", + }, { + Value: "s3.mel01.objectstorage.service.networklayer.com", + Help: "Melbourne Single Site Private Endpoint", + }, { + Value: "s3.tor01.objectstorage.softlayer.net", + Help: "Toronto Single Site Endpoint", + }, { + Value: "s3.tor01.objectstorage.service.networklayer.com", + Help: "Toronto Single Site Private Endpoint", + }}, + }, { + Name: "endpoint", + Help: "Endpoint for S3 API.\nRequired when using an S3 clone.", + Provider: "!AWS,IBMCOS", + Examples: []fs.OptionExample{{ + Value: "objects-us-west-1.dream.io", + Help: "Dream Objects endpoint", + Provider: "Dreamhost", + }, { + Value: "nyc3.digitaloceanspaces.com", + Help: "Digital Ocean Spaces New York 3", + Provider: "DigitalOcean", + }, { + Value: "ams3.digitaloceanspaces.com", + Help: "Digital Ocean Spaces Amsterdam 3", + Provider: "DigitalOcean", + }, { + Value: "sgp1.digitaloceanspaces.com", + Help: "Digital Ocean Spaces Singapore 1", + Provider: "DigitalOcean", + }, { + Value: "s3.wasabisys.com", + Help: "Wasabi Object Storage", + Provider: "Wasabi", + }}, + }, { + Name: "location_constraint", + Help: "Location constraint - must be set to match the Region.\nUsed when creating buckets only.", + Provider: "AWS", + Examples: []fs.OptionExample{{ + Value: "", + Help: "Empty for US Region, Northern Virginia or Pacific Northwest.", + }, { + Value: "us-east-2", + Help: "US East (Ohio) Region.", + }, { + Value: "us-west-2", + Help: "US West (Oregon) Region.", + }, { + Value: "us-west-1", + Help: "US West (Northern California) Region.", + }, { + Value: "ca-central-1", + Help: "Canada (Central) Region.", + }, { + Value: "eu-west-1", + Help: "EU (Ireland) Region.", + }, { + Value: "eu-west-2", + Help: "EU (London) Region.", + }, { + Value: "EU", + Help: "EU Region.", + }, { + Value: "ap-southeast-1", + Help: "Asia Pacific (Singapore) Region.", + }, { + Value: "ap-southeast-2", + Help: "Asia Pacific (Sydney) Region.", + }, { + Value: "ap-northeast-1", + Help: "Asia Pacific (Tokyo) Region.", + }, { + Value: "ap-northeast-2", + Help: "Asia Pacific (Seoul)", + }, { + Value: "ap-south-1", + Help: "Asia Pacific (Mumbai)", + }, { + Value: "sa-east-1", + Help: "South America (Sao Paulo) Region.", + }}, + }, { + Name: "location_constraint", + Help: "Location constraint - must match endpoint when using IBM Cloud Public.\nFor on-prem COS, do not make a selection from this list, hit enter", + Provider: "IBMCOS", + Examples: []fs.OptionExample{{ + Value: "us-standard", + Help: "US Cross Region Standard", + }, { + Value: "us-vault", + Help: "US Cross Region Vault", + }, { + Value: "us-cold", + Help: "US Cross Region Cold", + }, { + Value: "us-flex", + Help: "US Cross Region Flex", + }, { + Value: "us-east-standard", + Help: "US East Region Standard", + }, { + Value: "us-east-vault", + Help: "US East Region Vault", + }, { + Value: "us-east-cold", + Help: "US East Region Cold", + }, { + Value: "us-east-flex", + Help: "US East Region Flex", + }, { + Value: "us-south-standard", + Help: "US Sout hRegion Standard", + }, { + Value: "us-south-vault", + Help: "US South Region Vault", + }, { + Value: "us-south-cold", + Help: "US South Region Cold", + }, { + Value: "us-south-flex", + Help: "US South Region Flex", + }, { + Value: "eu-standard", + Help: "EU Cross Region Standard", + }, { + Value: "eu-vault", + Help: "EU Cross Region Vault", + }, { + Value: "eu-cold", + Help: "EU Cross Region Cold", + }, { + Value: "eu-flex", + Help: "EU Cross Region Flex", + }, { + Value: "eu-gb-standard", + Help: "Great Britan Standard", + }, { + Value: "eu-gb-vault", + Help: "Great Britan Vault", + }, { + Value: "eu-gb-cold", + Help: "Great Britan Cold", + }, { + Value: "eu-gb-flex", + Help: "Great Britan Flex", + }, { + Value: "ap-standard", + Help: "APAC Standard", + }, { + Value: "ap-vault", + Help: "APAC Vault", + }, { + Value: "ap-cold", + Help: "APAC Cold", + }, { + Value: "ap-flex", + Help: "APAC Flex", + }, { + Value: "mel01-standard", + Help: "Melbourne Standard", + }, { + Value: "mel01-vault", + Help: "Melbourne Vault", + }, { + Value: "mel01-cold", + Help: "Melbourne Cold", + }, { + Value: "mel01-flex", + Help: "Melbourne Flex", + }, { + Value: "tor01-standard", + Help: "Toronto Standard", + }, { + Value: "tor01-vault", + Help: "Toronto Vault", + }, { + Value: "tor01-cold", + Help: "Toronto Cold", + }, { + Value: "tor01-flex", + Help: "Toronto Flex", + }}, + }, { + Name: "location_constraint", + Help: "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.", + Provider: "!AWS,IBMCOS", + }, { + Name: "acl", + Help: "Canned ACL used when creating buckets and/or storing objects in S3.\nFor more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl", + Examples: []fs.OptionExample{{ + Value: "private", + Help: "Owner gets FULL_CONTROL. No one else has access rights (default).", + Provider: "!IBMCOS", + }, { + Value: "public-read", + Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ access.", + Provider: "!IBMCOS", + }, { + Value: "public-read-write", + Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended.", + Provider: "!IBMCOS", + }, { + Value: "authenticated-read", + Help: "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access.", + Provider: "!IBMCOS", + }, { + Value: "bucket-owner-read", + Help: "Object owner gets FULL_CONTROL. Bucket owner gets READ access.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.", + Provider: "!IBMCOS", + }, { + Value: "bucket-owner-full-control", + Help: "Both the object owner and the bucket owner get FULL_CONTROL over the object.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.", + Provider: "!IBMCOS", + }, { + Value: "private", + Help: "Owner gets FULL_CONTROL. No one else has access rights (default). This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise COS", + Provider: "IBMCOS", + }, { + Value: "public-read", + Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ access. This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise IBM COS", + Provider: "IBMCOS", + }, { + Value: "public-read-write", + Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. This acl is available on IBM Cloud (Infra), On-Premise IBM COS", + Provider: "IBMCOS", + }, { + Value: "authenticated-read", + Help: "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS", + Provider: "IBMCOS", + }}, + }, { + Name: "server_side_encryption", + Help: "The server-side encryption algorithm used when storing this object in S3.", + Provider: "AWS", + Examples: []fs.OptionExample{{ + Value: "", + Help: "None", + }, { + Value: "AES256", + Help: "AES256", + }, { + Value: "aws:kms", + Help: "aws:kms", + }}, + }, { + Name: "sse_kms_key_id", + Help: "If using KMS ID you must provide the ARN of Key.", + Provider: "AWS", + Examples: []fs.OptionExample{{ + Value: "", + Help: "None", + }, { + Value: "arn:aws:kms:us-east-1:*", + Help: "arn:aws:kms:*", + }}, + }, { + Name: "storage_class", + Help: "The storage class to use when storing objects in S3.", + Provider: "AWS", + Examples: []fs.OptionExample{{ + Value: "", + Help: "Default", + }, { + Value: "STANDARD", + Help: "Standard storage class", + }, { + Value: "REDUCED_REDUNDANCY", + Help: "Reduced redundancy storage class", + }, { + Value: "STANDARD_IA", + Help: "Standard Infrequent Access storage class", + }, { + Value: "ONEZONE_IA", + Help: "One Zone Infrequent Access storage class", + }}, + }, { + Name: "chunk_size", + Help: "Chunk size to use for uploading", + Default: fs.SizeSuffix(s3manager.MinUploadPartSize), + Advanced: true, + }, { + Name: "disable_checksum", + Help: "Don't store MD5 checksum with object metadata", + Default: false, + Advanced: true, + }, { + Name: "session_token", + Help: "An AWS session token", + Hide: fs.OptionHideBoth, + Advanced: true, + }, { + Name: "upload_concurrency", + Help: "Concurrency for multipart uploads.", + Default: 2, + Advanced: true, + }, { + Name: "force_path_style", + Help: "If true use path style access if false use virtual hosted style.\nSome providers (eg Aliyun OSS or Netease COS) require this.", + Default: true, + Advanced: true, + }}, + }) +} + +// Constants +const ( + metaMtime = "Mtime" // the meta key to store mtime in - eg X-Amz-Meta-Mtime + metaMD5Hash = "Md5chksum" // the meta key to store md5hash in + listChunkSize = 1000 // number of items to read at once + maxRetries = 10 // number of retries to make of operations + maxSizeForCopy = 5 * 1024 * 1024 * 1024 // The maximum size of object we can COPY + maxFileSize = 5 * 1024 * 1024 * 1024 * 1024 // largest possible upload file size +) + +// Options defines the configuration for this backend +type Options struct { + Provider string `config:"provider"` + EnvAuth bool `config:"env_auth"` + AccessKeyID string `config:"access_key_id"` + SecretAccessKey string `config:"secret_access_key"` + Region string `config:"region"` + Endpoint string `config:"endpoint"` + LocationConstraint string `config:"location_constraint"` + ACL string `config:"acl"` + ServerSideEncryption string `config:"server_side_encryption"` + SSEKMSKeyID string `config:"sse_kms_key_id"` + StorageClass string `config:"storage_class"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` + DisableChecksum bool `config:"disable_checksum"` + SessionToken string `config:"session_token"` + UploadConcurrency int `config:"upload_concurrency"` + ForcePathStyle bool `config:"force_path_style"` +} + +// Fs represents a remote s3 server +type Fs struct { + name string // the name of the remote + root string // root of the bucket - ignore all objects above this + opt Options // parsed options + features *fs.Features // optional features + c *s3.S3 // the connection to the s3 server + ses *session.Session // the s3 session + bucket string // the bucket we are working on + bucketOKMu sync.Mutex // mutex to protect bucket OK + bucketOK bool // true if we have created the bucket + bucketDeleted bool // true if we have deleted the bucket +} + +// Object describes a s3 object +type Object struct { + // Will definitely have everything but meta which may be nil + // + // List will read everything but meta & mimeType - to fill + // that in you need to call readMetaData + fs *Fs // what this object is part of + remote string // The remote path + etag string // md5sum of the object + bytes int64 // size of the object + lastModified time.Time // Last modified + meta map[string]*string // The object metadata if known - may be nil + mimeType string // MimeType of object - may be "" +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + if f.root == "" { + return f.bucket + } + return f.bucket + "/" + f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + if f.root == "" { + return fmt.Sprintf("S3 bucket %s", f.bucket) + } + return fmt.Sprintf("S3 bucket %s path %s", f.bucket, f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Pattern to match a s3 path +var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`) + +// parseParse parses a s3 'url' +func s3ParsePath(path string) (bucket, directory string, err error) { + parts := matcher.FindStringSubmatch(path) + if parts == nil { + err = errors.Errorf("couldn't parse bucket out of s3 path %q", path) + } else { + bucket, directory = parts[1], parts[2] + directory = strings.Trim(directory, "/") + } + return +} + +// s3Connection makes a connection to s3 +func s3Connection(opt *Options) (*s3.S3, *session.Session, error) { + // Make the auth + v := credentials.Value{ + AccessKeyID: opt.AccessKeyID, + SecretAccessKey: opt.SecretAccessKey, + SessionToken: opt.SessionToken, + } + + lowTimeoutClient := &http.Client{Timeout: 1 * time.Second} // low timeout to ec2 metadata service + def := defaults.Get() + def.Config.HTTPClient = lowTimeoutClient + + // first provider to supply a credential set "wins" + providers := []credentials.Provider{ + // use static credentials if they're present (checked by provider) + &credentials.StaticProvider{Value: v}, + + // * Access Key ID: AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY + // * Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY + &credentials.EnvProvider{}, + + // A SharedCredentialsProvider retrieves credentials + // from the current user's home directory. It checks + // AWS_SHARED_CREDENTIALS_FILE and AWS_PROFILE too. + &credentials.SharedCredentialsProvider{}, + + // Pick up IAM role if we're in an ECS task + defaults.RemoteCredProvider(*def.Config, def.Handlers), + + // Pick up IAM role in case we're on EC2 + &ec2rolecreds.EC2RoleProvider{ + Client: ec2metadata.New(session.New(), &aws.Config{ + HTTPClient: lowTimeoutClient, + }), + ExpiryWindow: 3, + }, + } + cred := credentials.NewChainCredentials(providers) + + switch { + case opt.EnvAuth: + // No need for empty checks if "env_auth" is true + case v.AccessKeyID == "" && v.SecretAccessKey == "": + // if no access key/secret and iam is explicitly disabled then fall back to anon interaction + cred = credentials.AnonymousCredentials + case v.AccessKeyID == "": + return nil, nil, errors.New("access_key_id not found") + case v.SecretAccessKey == "": + return nil, nil, errors.New("secret_access_key not found") + } + + if opt.Region == "" && opt.Endpoint == "" { + opt.Endpoint = "https://s3.amazonaws.com/" + } + if opt.Region == "" { + opt.Region = "us-east-1" + } + awsConfig := aws.NewConfig(). + WithRegion(opt.Region). + WithMaxRetries(maxRetries). + WithCredentials(cred). + WithEndpoint(opt.Endpoint). + WithHTTPClient(fshttp.NewClient(fs.Config)). + WithS3ForcePathStyle(opt.ForcePathStyle) + // awsConfig.WithLogLevel(aws.LogDebugWithSigning) + ses := session.New() + c := s3.New(ses, awsConfig) + if opt.Region == "other-v2-signature" { + fs.Debugf(nil, "Using v2 auth") + signer := func(req *request.Request) { + // Ignore AnonymousCredentials object + if req.Config.Credentials == credentials.AnonymousCredentials { + return + } + sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest) + } + c.Handlers.Sign.Clear() + c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler) + c.Handlers.Sign.PushBack(signer) + } + return c, ses, nil +} + +// NewFs constructs an Fs from the path, bucket:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + if opt.ChunkSize < fs.SizeSuffix(s3manager.MinUploadPartSize) { + return nil, errors.Errorf("s3 chunk size (%v) must be >= %v", opt.ChunkSize, fs.SizeSuffix(s3manager.MinUploadPartSize)) + } + bucket, directory, err := s3ParsePath(root) + if err != nil { + return nil, err + } + c, ses, err := s3Connection(opt) + if err != nil { + return nil, err + } + f := &Fs{ + name: name, + root: directory, + opt: *opt, + c: c, + bucket: bucket, + ses: ses, + } + f.features = (&fs.Features{ + ReadMimeType: true, + WriteMimeType: true, + BucketBased: true, + }).Fill(f) + if f.root != "" { + f.root += "/" + // Check to see if the object exists + req := s3.HeadObjectInput{ + Bucket: &f.bucket, + Key: &directory, + } + _, err = f.c.HeadObject(&req) + if err == nil { + f.root = path.Dir(directory) + if f.root == "." { + f.root = "" + } else { + f.root += "/" + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + } + // f.listMultipartUploads() + return f, nil +} + +// Return an Object from a path +// +//If it can't be found it returns the error ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *s3.Object) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + if info != nil { + // Set info but not meta + if info.LastModified == nil { + fs.Logf(o, "Failed to read last modified") + o.lastModified = time.Now() + } else { + o.lastModified = *info.LastModified + } + o.etag = aws.StringValue(info.ETag) + o.bytes = aws.Int64Value(info.Size) + } else { + err := o.readMetaData() // reads info and meta, returning an error + if err != nil { + return nil, err + } + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// listFn is called from list to handle an object. +type listFn func(remote string, object *s3.Object, isDirectory bool) error + +// list the objects into the function supplied +// +// dir is the starting directory, "" for root +// +// Set recurse to read sub directories +func (f *Fs) list(dir string, recurse bool, fn listFn) error { + root := f.root + if dir != "" { + root += dir + "/" + } + maxKeys := int64(listChunkSize) + delimiter := "" + if !recurse { + delimiter = "/" + } + var marker *string + for { + // FIXME need to implement ALL loop + req := s3.ListObjectsInput{ + Bucket: &f.bucket, + Delimiter: &delimiter, + Prefix: &root, + MaxKeys: &maxKeys, + Marker: marker, + } + resp, err := f.c.ListObjects(&req) + if err != nil { + if awsErr, ok := err.(awserr.RequestFailure); ok { + if awsErr.StatusCode() == http.StatusNotFound { + err = fs.ErrorDirNotFound + } + } + return err + } + rootLength := len(f.root) + if !recurse { + for _, commonPrefix := range resp.CommonPrefixes { + if commonPrefix.Prefix == nil { + fs.Logf(f, "Nil common prefix received") + continue + } + remote := *commonPrefix.Prefix + if !strings.HasPrefix(remote, f.root) { + fs.Logf(f, "Odd name received %q", remote) + continue + } + remote = remote[rootLength:] + if strings.HasSuffix(remote, "/") { + remote = remote[:len(remote)-1] + } + err = fn(remote, &s3.Object{Key: &remote}, true) + if err != nil { + return err + } + } + } + for _, object := range resp.Contents { + key := aws.StringValue(object.Key) + if !strings.HasPrefix(key, f.root) { + fs.Logf(f, "Odd name received %q", key) + continue + } + remote := key[rootLength:] + // is this a directory marker? + if (strings.HasSuffix(remote, "/") || remote == "") && *object.Size == 0 { + if recurse && remote != "" { + // add a directory in if --fast-list since will have no prefixes + remote = remote[:len(remote)-1] + err = fn(remote, &s3.Object{Key: &remote}, true) + if err != nil { + return err + } + } + continue // skip directory marker + } + err = fn(remote, object, false) + if err != nil { + return err + } + } + if !aws.BoolValue(resp.IsTruncated) { + break + } + // Use NextMarker if set, otherwise use last Key + if resp.NextMarker == nil || *resp.NextMarker == "" { + if len(resp.Contents) == 0 { + return errors.New("s3 protocol error: received listing with IsTruncated set, no NextMarker and no Contents") + } + marker = resp.Contents[len(resp.Contents)-1].Key + } else { + marker = resp.NextMarker + } + } + return nil +} + +// Convert a list item into a DirEntry +func (f *Fs) itemToDirEntry(remote string, object *s3.Object, isDirectory bool) (fs.DirEntry, error) { + if isDirectory { + size := int64(0) + if object.Size != nil { + size = *object.Size + } + d := fs.NewDir(remote, time.Time{}).SetSize(size) + return d, nil + } + o, err := f.newObjectWithInfo(remote, object) + if err != nil { + return nil, err + } + return o, nil +} + +// mark the bucket as being OK +func (f *Fs) markBucketOK() { + if f.bucket != "" { + f.bucketOKMu.Lock() + f.bucketOK = true + f.bucketDeleted = false + f.bucketOKMu.Unlock() + } +} + +// listDir lists files and directories to out +func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { + // List the objects and directories + err = f.list(dir, false, func(remote string, object *s3.Object, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + if entry != nil { + entries = append(entries, entry) + } + return nil + }) + if err != nil { + return nil, err + } + // bucket must be present if listing succeeded + f.markBucketOK() + return entries, nil +} + +// listBuckets lists the buckets to out +func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) { + if dir != "" { + return nil, fs.ErrorListBucketRequired + } + req := s3.ListBucketsInput{} + resp, err := f.c.ListBuckets(&req) + if err != nil { + return nil, err + } + for _, bucket := range resp.Buckets { + d := fs.NewDir(aws.StringValue(bucket.Name), aws.TimeValue(bucket.CreationDate)) + entries = append(entries, d) + } + return entries, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + if f.bucket == "" { + return f.listBuckets(dir) + } + return f.listDir(dir) +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + if f.bucket == "" { + return fs.ErrorListBucketRequired + } + list := walk.NewListRHelper(callback) + err = f.list(dir, true, func(remote string, object *s3.Object, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + return list.Add(entry) + }) + if err != nil { + return err + } + // bucket must be present if listing succeeded + f.markBucketOK() + return list.Flush() +} + +// Put the Object into the bucket +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + // Temporary Object under construction + fs := &Object{ + fs: f, + remote: src.Remote(), + } + return fs, fs.Update(in, src, options...) +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// Check if the bucket exists +// +// NB this can return incorrect results if called immediately after bucket deletion +func (f *Fs) dirExists() (bool, error) { + req := s3.HeadBucketInput{ + Bucket: &f.bucket, + } + _, err := f.c.HeadBucket(&req) + if err == nil { + return true, nil + } + if err, ok := err.(awserr.RequestFailure); ok { + if err.StatusCode() == http.StatusNotFound { + return false, nil + } + } + return false, err +} + +// Mkdir creates the bucket if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + f.bucketOKMu.Lock() + defer f.bucketOKMu.Unlock() + if f.bucketOK { + return nil + } + if !f.bucketDeleted { + exists, err := f.dirExists() + if err == nil { + f.bucketOK = exists + } + if err != nil || exists { + return err + } + } + req := s3.CreateBucketInput{ + Bucket: &f.bucket, + ACL: &f.opt.ACL, + } + if f.opt.LocationConstraint != "" { + req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{ + LocationConstraint: &f.opt.LocationConstraint, + } + } + _, err := f.c.CreateBucket(&req) + if err, ok := err.(awserr.Error); ok { + if err.Code() == "BucketAlreadyOwnedByYou" { + err = nil + } + } + if err == nil { + f.bucketOK = true + f.bucketDeleted = false + } + return err +} + +// Rmdir deletes the bucket if the fs is at the root +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + f.bucketOKMu.Lock() + defer f.bucketOKMu.Unlock() + if f.root != "" || dir != "" { + return nil + } + req := s3.DeleteBucketInput{ + Bucket: &f.bucket, + } + _, err := f.c.DeleteBucket(&req) + if err == nil { + f.bucketOK = false + f.bucketDeleted = true + } + return err +} + +// Precision of the remote +func (f *Fs) Precision() time.Duration { + return time.Nanosecond +} + +// pathEscape escapes s as for a URL path. It uses rest.URLPathEscape +// but also escapes '+' for S3 and Digital Ocean spaces compatibility +func pathEscape(s string) string { + return strings.Replace(rest.URLPathEscape(s), "+", "%2B", -1) +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + err := f.Mkdir("") + if err != nil { + return nil, err + } + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + srcFs := srcObj.fs + key := f.root + remote + source := pathEscape(srcFs.bucket + "/" + srcFs.root + srcObj.remote) + req := s3.CopyObjectInput{ + Bucket: &f.bucket, + Key: &key, + CopySource: &source, + MetadataDirective: aws.String(s3.MetadataDirectiveCopy), + } + _, err = f.c.CopyObject(&req) + if err != nil { + return nil, err + } + return f.NewObject(remote) +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`) + +// Hash returns the Md5sum of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.MD5 { + return "", hash.ErrUnsupported + } + hash := strings.Trim(strings.ToLower(o.etag), `"`) + // Check the etag is a valid md5sum + if !matchMd5.MatchString(hash) { + err := o.readMetaData() + if err != nil { + return "", err + } + + if md5sum, ok := o.meta[metaMD5Hash]; ok { + md5sumBytes, err := base64.StdEncoding.DecodeString(*md5sum) + if err != nil { + return "", err + } + hash = hex.EncodeToString(md5sumBytes) + } else { + hash = "" + } + } + return hash, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.bytes +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData() (err error) { + if o.meta != nil { + return nil + } + key := o.fs.root + o.remote + req := s3.HeadObjectInput{ + Bucket: &o.fs.bucket, + Key: &key, + } + resp, err := o.fs.c.HeadObject(&req) + if err != nil { + if awsErr, ok := err.(awserr.RequestFailure); ok { + if awsErr.StatusCode() == http.StatusNotFound { + return fs.ErrorObjectNotFound + } + } + return err + } + var size int64 + // Ignore missing Content-Length assuming it is 0 + // Some versions of ceph do this due their apache proxies + if resp.ContentLength != nil { + size = *resp.ContentLength + } + o.etag = aws.StringValue(resp.ETag) + o.bytes = size + o.meta = resp.Metadata + if resp.LastModified == nil { + fs.Logf(o, "Failed to read last modified from HEAD: %v", err) + o.lastModified = time.Now() + } else { + o.lastModified = *resp.LastModified + } + o.mimeType = aws.StringValue(resp.ContentType) + return nil +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + if fs.Config.UseServerModTime { + return o.lastModified + } + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + // read mtime out of metadata if available + d, ok := o.meta[metaMtime] + if !ok || d == nil { + // fs.Debugf(o, "No metadata") + return o.lastModified + } + modTime, err := swift.FloatStringToTime(*d) + if err != nil { + fs.Logf(o, "Failed to read mtime from object: %v", err) + return o.lastModified + } + return modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + err := o.readMetaData() + if err != nil { + return err + } + o.meta[metaMtime] = aws.String(swift.TimeToFloatString(modTime)) + + if o.bytes >= maxSizeForCopy { + fs.Debugf(o, "SetModTime is unsupported for objects bigger than %v bytes", fs.SizeSuffix(maxSizeForCopy)) + return nil + } + + // Guess the content type + mimeType := fs.MimeType(o) + + // Copy the object to itself to update the metadata + key := o.fs.root + o.remote + sourceKey := o.fs.bucket + "/" + key + directive := s3.MetadataDirectiveReplace // replace metadata with that passed in + req := s3.CopyObjectInput{ + Bucket: &o.fs.bucket, + ACL: &o.fs.opt.ACL, + Key: &key, + ContentType: &mimeType, + CopySource: aws.String(pathEscape(sourceKey)), + Metadata: o.meta, + MetadataDirective: &directive, + } + _, err = o.fs.c.CopyObject(&req) + return err +} + +// Storable raturns a boolean indicating if this object is storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + key := o.fs.root + o.remote + req := s3.GetObjectInput{ + Bucket: &o.fs.bucket, + Key: &key, + } + for _, option := range options { + switch option.(type) { + case *fs.RangeOption, *fs.SeekOption: + _, value := option.Header() + req.Range = &value + default: + if option.Mandatory() { + fs.Logf(o, "Unsupported mandatory option: %v", option) + } + } + } + resp, err := o.fs.c.GetObject(&req) + if err, ok := err.(awserr.RequestFailure); ok { + if err.Code() == "InvalidObjectState" { + return nil, errors.Errorf("Object in GLACIER, restore first: %v", key) + } + } + if err != nil { + return nil, err + } + return resp.Body, nil +} + +// Update the Object from in with modTime and size +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + err := o.fs.Mkdir("") + if err != nil { + return err + } + modTime := src.ModTime() + size := src.Size() + + uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) { + u.Concurrency = o.fs.opt.UploadConcurrency + u.LeavePartsOnError = false + u.S3 = o.fs.c + u.PartSize = int64(o.fs.opt.ChunkSize) + + if size == -1 { + // Make parts as small as possible while still being able to upload to the + // S3 file size limit. Rounded up to nearest MB. + u.PartSize = (((maxFileSize / s3manager.MaxUploadParts) >> 20) + 1) << 20 + return + } + // Adjust PartSize until the number of parts is small enough. + if size/u.PartSize >= s3manager.MaxUploadParts { + // Calculate partition size rounded up to the nearest MB + u.PartSize = (((size / s3manager.MaxUploadParts) >> 20) + 1) << 20 + } + }) + + // Set the mtime in the meta data + metadata := map[string]*string{ + metaMtime: aws.String(swift.TimeToFloatString(modTime)), + } + + if !o.fs.opt.DisableChecksum && size > uploader.PartSize { + hash, err := src.Hash(hash.MD5) + + if err == nil && matchMd5.MatchString(hash) { + hashBytes, err := hex.DecodeString(hash) + + if err == nil { + metadata[metaMD5Hash] = aws.String(base64.StdEncoding.EncodeToString(hashBytes)) + } + } + } + + // Guess the content type + mimeType := fs.MimeType(src) + + key := o.fs.root + o.remote + req := s3manager.UploadInput{ + Bucket: &o.fs.bucket, + ACL: &o.fs.opt.ACL, + Key: &key, + Body: in, + ContentType: &mimeType, + Metadata: metadata, + //ContentLength: &size, + } + if o.fs.opt.ServerSideEncryption != "" { + req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption + } + if o.fs.opt.SSEKMSKeyID != "" { + req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID + } + if o.fs.opt.StorageClass != "" { + req.StorageClass = &o.fs.opt.StorageClass + } + _, err = uploader.Upload(&req) + if err != nil { + return err + } + + // Read the metadata from the newly created object + o.meta = nil // wipe old metadata + err = o.readMetaData() + return err +} + +// Remove an object +func (o *Object) Remove() error { + key := o.fs.root + o.remote + req := s3.DeleteObjectInput{ + Bucket: &o.fs.bucket, + Key: &key, + } + _, err := o.fs.c.DeleteObject(&req) + return err +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return "" + } + return o.mimeType +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.Copier = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.ListRer = &Fs{} + _ fs.Object = &Object{} + _ fs.MimeTyper = &Object{} +) diff --git a/.rclone_repo/backend/s3/s3_test.go b/.rclone_repo/backend/s3/s3_test.go new file mode 100755 index 0000000..d1d2d4c --- /dev/null +++ b/.rclone_repo/backend/s3/s3_test.go @@ -0,0 +1,17 @@ +// Test S3 filesystem interface +package s3_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/s3" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestS3:", + NilObject: (*s3.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/s3/v2sign.go b/.rclone_repo/backend/s3/v2sign.go new file mode 100755 index 0000000..23048ed --- /dev/null +++ b/.rclone_repo/backend/s3/v2sign.go @@ -0,0 +1,115 @@ +// v2 signing + +package s3 + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "net/http" + "sort" + "strings" + "time" +) + +// URL parameters that need to be added to the signature +var s3ParamsToSign = map[string]struct{}{ + "acl": {}, + "location": {}, + "logging": {}, + "notification": {}, + "partNumber": {}, + "policy": {}, + "requestPayment": {}, + "torrent": {}, + "uploadId": {}, + "uploads": {}, + "versionId": {}, + "versioning": {}, + "versions": {}, + "response-content-type": {}, + "response-content-language": {}, + "response-expires": {}, + "response-cache-control": {}, + "response-content-disposition": {}, + "response-content-encoding": {}, +} + +// sign signs requests using v2 auth +// +// Cobbled together from goamz and aws-sdk-go +func sign(AccessKey, SecretKey string, req *http.Request) { + // Set date + date := time.Now().UTC().Format(time.RFC1123) + req.Header.Set("Date", date) + + // Sort out URI + uri := req.URL.Opaque + if uri != "" { + if strings.HasPrefix(uri, "//") { + // Strip off //host/uri + uri = "/" + strings.Join(strings.Split(uri, "/")[3:], "/") + req.URL.Opaque = uri // reset to plain URI otherwise Ceph gets confused + } + } else { + uri = req.URL.Path + } + if uri == "" { + uri = "/" + } + + // Look through headers of interest + var md5 string + var contentType string + var headersToSign []string + for k, v := range req.Header { + k = strings.ToLower(k) + switch k { + case "content-md5": + md5 = v[0] + case "content-type": + contentType = v[0] + default: + if strings.HasPrefix(k, "x-amz-") { + vall := strings.Join(v, ",") + headersToSign = append(headersToSign, k+":"+vall) + } + } + } + // Make headers of interest into canonical string + var joinedHeadersToSign string + if len(headersToSign) > 0 { + sort.StringSlice(headersToSign).Sort() + joinedHeadersToSign = strings.Join(headersToSign, "\n") + "\n" + } + + // Look for query parameters which need to be added to the signature + params := req.URL.Query() + var queriesToSign []string + for k, vs := range params { + if _, ok := s3ParamsToSign[k]; ok { + for _, v := range vs { + if v == "" { + queriesToSign = append(queriesToSign, k) + } else { + queriesToSign = append(queriesToSign, k+"="+v) + } + } + } + } + // Add query parameters to URI + if len(queriesToSign) > 0 { + sort.StringSlice(queriesToSign).Sort() + uri += "?" + strings.Join(queriesToSign, "&") + } + + // Make signature + payload := req.Method + "\n" + md5 + "\n" + contentType + "\n" + date + "\n" + joinedHeadersToSign + uri + hash := hmac.New(sha1.New, []byte(SecretKey)) + _, _ = hash.Write([]byte(payload)) + signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size())) + base64.StdEncoding.Encode(signature, hash.Sum(nil)) + + // Set signature in request + req.Header.Set("Authorization", "AWS "+AccessKey+":"+string(signature)) +} diff --git a/.rclone_repo/backend/sftp/sftp.go b/.rclone_repo/backend/sftp/sftp.go new file mode 100755 index 0000000..d27a43c --- /dev/null +++ b/.rclone_repo/backend/sftp/sftp.go @@ -0,0 +1,1032 @@ +// Package sftp provides a filesystem interface using github.com/pkg/sftp + +// +build !plan9,go1.9 + +package sftp + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "os" + "os/user" + "path" + "regexp" + "strings" + "sync" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/readers" + "github.com/pkg/errors" + "github.com/pkg/sftp" + "github.com/xanzy/ssh-agent" + "golang.org/x/crypto/ssh" + "golang.org/x/time/rate" +) + +const ( + connectionsPerSecond = 10 // don't make more than this many ssh connections/s +) + +var ( + currentUser = readCurrentUser() +) + +func init() { + fsi := &fs.RegInfo{ + Name: "sftp", + Description: "SSH/SFTP Connection", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "host", + Help: "SSH host to connect to", + Required: true, + Examples: []fs.OptionExample{{ + Value: "example.com", + Help: "Connect to example.com", + }}, + }, { + Name: "user", + Help: "SSH username, leave blank for current username, " + currentUser, + }, { + Name: "port", + Help: "SSH port, leave blank to use default (22)", + }, { + Name: "pass", + Help: "SSH password, leave blank to use ssh-agent.", + IsPassword: true, + }, { + Name: "key_file", + Help: "Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.", + }, { + Name: "use_insecure_cipher", + Help: "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.", + Default: false, + Examples: []fs.OptionExample{ + { + Value: "false", + Help: "Use default Cipher list.", + }, { + Value: "true", + Help: "Enables the use of the aes128-cbc cipher.", + }, + }, + }, { + Name: "disable_hashcheck", + Default: false, + Help: "Disable the execution of SSH commands to determine if remote file hashing is available.\nLeave blank or set to false to enable hashing (recommended), set to true to disable hashing.", + }, { + Name: "ask_password", + Default: false, + Help: "Allow asking for SFTP password when needed.", + Advanced: true, + }, { + Name: "path_override", + Default: "", + Help: "Override path used by SSH connection.", + Advanced: true, + }, { + Name: "set_modtime", + Default: true, + Help: "Set the modified time on the remote if set.", + Advanced: true, + }}, + } + fs.Register(fsi) +} + +// Options defines the configuration for this backend +type Options struct { + Host string `config:"host"` + User string `config:"user"` + Port string `config:"port"` + Pass string `config:"pass"` + KeyFile string `config:"key_file"` + UseInsecureCipher bool `config:"use_insecure_cipher"` + DisableHashCheck bool `config:"disable_hashcheck"` + AskPassword bool `config:"ask_password"` + PathOverride string `config:"path_override"` + SetModTime bool `config:"set_modtime"` +} + +// Fs stores the interface to the remote SFTP files +type Fs struct { + name string + root string + opt Options // parsed options + features *fs.Features // optional features + config *ssh.ClientConfig + url string + mkdirLock *stringLock + cachedHashes *hash.Set + poolMu sync.Mutex + pool []*conn + connLimit *rate.Limiter // for limiting number of connections per second +} + +// Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading) +type Object struct { + fs *Fs + remote string + size int64 // size of the object + modTime time.Time // modification time of the object + mode os.FileMode // mode bits from the file + md5sum *string // Cached MD5 checksum + sha1sum *string // Cached SHA1 checksum +} + +// readCurrentUser finds the current user name or "" if not found +func readCurrentUser() (userName string) { + usr, err := user.Current() + if err == nil { + return usr.Username + } + // Fall back to reading $USER then $LOGNAME + userName = os.Getenv("USER") + if userName != "" { + return userName + } + return os.Getenv("LOGNAME") +} + +// Dial starts a client connection to the given SSH server. It is a +// convenience function that connects to the given network address, +// initiates the SSH handshake, and then sets up a Client. +func Dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) { + dialer := fshttp.NewDialer(fs.Config) + conn, err := dialer.Dial(network, addr) + if err != nil { + return nil, err + } + c, chans, reqs, err := ssh.NewClientConn(conn, addr, sshConfig) + if err != nil { + return nil, err + } + return ssh.NewClient(c, chans, reqs), nil +} + +// conn encapsulates an ssh client and corresponding sftp client +type conn struct { + sshClient *ssh.Client + sftpClient *sftp.Client + err chan error +} + +// Wait for connection to close +func (c *conn) wait() { + c.err <- c.sshClient.Conn.Wait() +} + +// Closes the connection +func (c *conn) close() error { + sftpErr := c.sftpClient.Close() + sshErr := c.sshClient.Close() + if sftpErr != nil { + return sftpErr + } + return sshErr +} + +// Returns an error if closed +func (c *conn) closed() error { + select { + case err := <-c.err: + return err + default: + } + return nil +} + +// Open a new connection to the SFTP server. +func (f *Fs) sftpConnection() (c *conn, err error) { + // Rate limit rate of new connections + err = f.connLimit.Wait(context.Background()) + if err != nil { + return nil, errors.Wrap(err, "limiter failed in connect") + } + c = &conn{ + err: make(chan error, 1), + } + c.sshClient, err = Dial("tcp", f.opt.Host+":"+f.opt.Port, f.config) + if err != nil { + return nil, errors.Wrap(err, "couldn't connect SSH") + } + c.sftpClient, err = sftp.NewClient(c.sshClient) + if err != nil { + _ = c.sshClient.Close() + return nil, errors.Wrap(err, "couldn't initialise SFTP") + } + go c.wait() + return c, nil +} + +// Get an SFTP connection from the pool, or open a new one +func (f *Fs) getSftpConnection() (c *conn, err error) { + f.poolMu.Lock() + for len(f.pool) > 0 { + c = f.pool[0] + f.pool = f.pool[1:] + err := c.closed() + if err == nil { + break + } + fs.Errorf(f, "Discarding closed SSH connection: %v", err) + c = nil + } + f.poolMu.Unlock() + if c != nil { + return c, nil + } + return f.sftpConnection() +} + +// Return an SFTP connection to the pool +// +// It nils the pointed to connection out so it can't be reused +// +// if err is not nil then it checks the connection is alive using a +// Getwd request +func (f *Fs) putSftpConnection(pc **conn, err error) { + c := *pc + *pc = nil + if err != nil { + // work out if this is an expected error + underlyingErr := errors.Cause(err) + isRegularError := false + switch underlyingErr { + case os.ErrNotExist: + isRegularError = true + default: + switch underlyingErr.(type) { + case *sftp.StatusError, *os.PathError: + isRegularError = true + } + } + // If not a regular SFTP error code then check the connection + if !isRegularError { + _, nopErr := c.sftpClient.Getwd() + if nopErr != nil { + fs.Debugf(f, "Connection failed, closing: %v", nopErr) + _ = c.close() + return + } + fs.Debugf(f, "Connection OK after error: %v", err) + } + } + f.poolMu.Lock() + f.pool = append(f.pool, c) + f.poolMu.Unlock() +} + +// NewFs creates a new Fs object from the name and root. It connects to +// the host specified in the config file. +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + if opt.User == "" { + opt.User = currentUser + } + if opt.Port == "" { + opt.Port = "22" + } + sshConfig := &ssh.ClientConfig{ + User: opt.User, + Auth: []ssh.AuthMethod{}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: fs.Config.ConnectTimeout, + } + + if opt.UseInsecureCipher { + sshConfig.Config.SetDefaults() + sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc") + } + + // Add ssh agent-auth if no password or file specified + if opt.Pass == "" && opt.KeyFile == "" { + sshAgentClient, _, err := sshagent.New() + if err != nil { + return nil, errors.Wrap(err, "couldn't connect to ssh-agent") + } + signers, err := sshAgentClient.Signers() + if err != nil { + return nil, errors.Wrap(err, "couldn't read ssh agent signers") + } + sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...)) + } + + // Load key file if specified + if opt.KeyFile != "" { + key, err := ioutil.ReadFile(opt.KeyFile) + if err != nil { + return nil, errors.Wrap(err, "failed to read private key file") + } + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + return nil, errors.Wrap(err, "failed to parse private key file") + } + sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signer)) + } + + // Auth from password if specified + if opt.Pass != "" { + clearpass, err := obscure.Reveal(opt.Pass) + if err != nil { + return nil, err + } + sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass)) + } + + // Ask for password if none was defined and we're allowed to + if opt.Pass == "" && opt.AskPassword { + _, _ = fmt.Fprint(os.Stderr, "Enter SFTP password: ") + clearpass := config.ReadPassword() + sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass)) + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + config: sshConfig, + url: "sftp://" + opt.User + "@" + opt.Host + ":" + opt.Port + "/" + root, + mkdirLock: newStringLock(), + connLimit: rate.NewLimiter(rate.Limit(connectionsPerSecond), 1), + } + f.features = (&fs.Features{ + CanHaveEmptyDirectories: true, + }).Fill(f) + // Make a connection and pool it to return errors early + c, err := f.getSftpConnection() + if err != nil { + return nil, errors.Wrap(err, "NewFs") + } + f.putSftpConnection(&c, nil) + if root != "" { + // Check to see if the root actually an existing file + remote := path.Base(root) + f.root = path.Dir(root) + if f.root == "." { + f.root = "" + } + _, err := f.NewObject(remote) + if err != nil { + if err == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile { + // File doesn't exist so return old f + f.root = root + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + return f, nil +} + +// Name returns the configured name of the file system +func (f *Fs) Name() string { + return f.name +} + +// Root returns the root for the filesystem +func (f *Fs) Root() string { + return f.root +} + +// String returns the URL for the filesystem +func (f *Fs) String() string { + return f.url +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Precision is the remote sftp file system's modtime precision, which we have no way of knowing. We estimate at 1s +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// NewObject creates a new remote sftp file object +func (f *Fs) NewObject(remote string) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + err := o.stat() + if err != nil { + return nil, err + } + return o, nil +} + +// dirExists returns true,nil if the directory exists, false, nil if +// it doesn't or false, err +func (f *Fs) dirExists(dir string) (bool, error) { + if dir == "" { + dir = "." + } + c, err := f.getSftpConnection() + if err != nil { + return false, errors.Wrap(err, "dirExists") + } + info, err := c.sftpClient.Stat(dir) + f.putSftpConnection(&c, err) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, errors.Wrap(err, "dirExists stat failed") + } + if !info.IsDir() { + return false, fs.ErrorIsFile + } + return true, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + root := path.Join(f.root, dir) + ok, err := f.dirExists(root) + if err != nil { + return nil, errors.Wrap(err, "List failed") + } + if !ok { + return nil, fs.ErrorDirNotFound + } + sftpDir := root + if sftpDir == "" { + sftpDir = "." + } + c, err := f.getSftpConnection() + if err != nil { + return nil, errors.Wrap(err, "List") + } + infos, err := c.sftpClient.ReadDir(sftpDir) + f.putSftpConnection(&c, err) + if err != nil { + return nil, errors.Wrapf(err, "error listing %q", dir) + } + for _, info := range infos { + remote := path.Join(dir, info.Name()) + // If file is a symlink (not a regular file is the best cross platform test we can do), do a stat to + // pick up the size and type of the destination, instead of the size and type of the symlink. + if !info.Mode().IsRegular() { + info, err = f.stat(remote) + if err != nil { + return nil, errors.Wrap(err, "stat of non-regular file/dir failed") + } + } + if info.IsDir() { + d := fs.NewDir(remote, info.ModTime()) + entries = append(entries, d) + } else { + o := &Object{ + fs: f, + remote: remote, + } + o.setMetadata(info) + entries = append(entries, o) + } + } + return entries, nil +} + +// Put data from into a new remote sftp file object described by and +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + err := f.mkParentDir(src.Remote()) + if err != nil { + return nil, errors.Wrap(err, "Put mkParentDir failed") + } + // Temporary object under construction + o := &Object{ + fs: f, + remote: src.Remote(), + } + err = o.Update(in, src, options...) + if err != nil { + return nil, err + } + return o, nil +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// mkParentDir makes the parent of remote if necessary and any +// directories above that +func (f *Fs) mkParentDir(remote string) error { + parent := path.Dir(remote) + return f.mkdir(path.Join(f.root, parent)) +} + +// mkdir makes the directory and parents using native paths +func (f *Fs) mkdir(dirPath string) error { + f.mkdirLock.Lock(dirPath) + defer f.mkdirLock.Unlock(dirPath) + if dirPath == "." || dirPath == "/" { + return nil + } + ok, err := f.dirExists(dirPath) + if err != nil { + return errors.Wrap(err, "mkdir dirExists failed") + } + if ok { + return nil + } + parent := path.Dir(dirPath) + err = f.mkdir(parent) + if err != nil { + return err + } + c, err := f.getSftpConnection() + if err != nil { + return errors.Wrap(err, "mkdir") + } + err = c.sftpClient.Mkdir(dirPath) + f.putSftpConnection(&c, err) + if err != nil { + return errors.Wrapf(err, "mkdir %q failed", dirPath) + } + return nil +} + +// Mkdir makes the root directory of the Fs object +func (f *Fs) Mkdir(dir string) error { + root := path.Join(f.root, dir) + return f.mkdir(root) +} + +// Rmdir removes the root directory of the Fs object +func (f *Fs) Rmdir(dir string) error { + root := path.Join(f.root, dir) + c, err := f.getSftpConnection() + if err != nil { + return errors.Wrap(err, "Rmdir") + } + err = c.sftpClient.Remove(root) + f.putSftpConnection(&c, err) + return err +} + +// Move renames a remote sftp file object +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + err := f.mkParentDir(remote) + if err != nil { + return nil, errors.Wrap(err, "Move mkParentDir failed") + } + c, err := f.getSftpConnection() + if err != nil { + return nil, errors.Wrap(err, "Move") + } + err = c.sftpClient.Rename( + srcObj.path(), + path.Join(f.root, remote), + ) + f.putSftpConnection(&c, err) + if err != nil { + return nil, errors.Wrap(err, "Move Rename failed") + } + dstObj, err := f.NewObject(remote) + if err != nil { + return nil, errors.Wrap(err, "Move NewObject failed") + } + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := path.Join(srcFs.root, srcRemote) + dstPath := path.Join(f.root, dstRemote) + + // Check if destination exists + ok, err := f.dirExists(dstPath) + if err != nil { + return errors.Wrap(err, "DirMove dirExists dst failed") + } + if ok { + return fs.ErrorDirExists + } + + // Make sure the parent directory exists + err = f.mkdir(path.Dir(dstPath)) + if err != nil { + return errors.Wrap(err, "DirMove mkParentDir dst failed") + } + + // Do the move + c, err := f.getSftpConnection() + if err != nil { + return errors.Wrap(err, "DirMove") + } + err = c.sftpClient.Rename( + srcPath, + dstPath, + ) + f.putSftpConnection(&c, err) + if err != nil { + return errors.Wrapf(err, "DirMove Rename(%q,%q) failed", srcPath, dstPath) + } + return nil +} + +// Hashes returns the supported hash types of the filesystem +func (f *Fs) Hashes() hash.Set { + if f.cachedHashes != nil { + return *f.cachedHashes + } + + if f.opt.DisableHashCheck { + return hash.Set(hash.None) + } + + c, err := f.getSftpConnection() + if err != nil { + fs.Errorf(f, "Couldn't get SSH connection to figure out Hashes: %v", err) + return hash.Set(hash.None) + } + defer f.putSftpConnection(&c, err) + session, err := c.sshClient.NewSession() + if err != nil { + return hash.Set(hash.None) + } + sha1Output, _ := session.Output("echo 'abc' | sha1sum") + expectedSha1 := "03cfd743661f07975fa2f1220c5194cbaff48451" + _ = session.Close() + + session, err = c.sshClient.NewSession() + if err != nil { + return hash.Set(hash.None) + } + md5Output, _ := session.Output("echo 'abc' | md5sum") + expectedMd5 := "0bee89b07a248e27c83fc3d5951213c1" + _ = session.Close() + + sha1Works := parseHash(sha1Output) == expectedSha1 + md5Works := parseHash(md5Output) == expectedMd5 + + set := hash.NewHashSet() + if !sha1Works && !md5Works { + set.Add(hash.None) + } + if sha1Works { + set.Add(hash.SHA1) + } + if md5Works { + set.Add(hash.MD5) + } + + _ = session.Close() + f.cachedHashes = &set + return set +} + +// Fs is the filesystem this remote sftp file object is located within +func (o *Object) Fs() fs.Info { + return o.fs +} + +// String returns the URL to the remote SFTP file +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote the name of the remote SFTP file, relative to the fs root +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the selected checksum of the file +// If no checksum is available it returns "" +func (o *Object) Hash(r hash.Type) (string, error) { + var hashCmd string + if r == hash.MD5 { + if o.md5sum != nil { + return *o.md5sum, nil + } + hashCmd = "md5sum" + } else if r == hash.SHA1 { + if o.sha1sum != nil { + return *o.sha1sum, nil + } + hashCmd = "sha1sum" + } else { + return "", hash.ErrUnsupported + } + + c, err := o.fs.getSftpConnection() + if err != nil { + return "", errors.Wrap(err, "Hash get SFTP connection") + } + session, err := c.sshClient.NewSession() + o.fs.putSftpConnection(&c, err) + if err != nil { + return "", errors.Wrap(err, "Hash put SFTP connection") + } + + var stdout, stderr bytes.Buffer + session.Stdout = &stdout + session.Stderr = &stderr + escapedPath := shellEscape(o.path()) + if o.fs.opt.PathOverride != "" { + escapedPath = shellEscape(path.Join(o.fs.opt.PathOverride, o.remote)) + } + err = session.Run(hashCmd + " " + escapedPath) + if err != nil { + _ = session.Close() + fs.Debugf(o, "Failed to calculate %v hash: %v (%s)", r, err, bytes.TrimSpace(stderr.Bytes())) + return "", nil + } + + _ = session.Close() + str := parseHash(stdout.Bytes()) + if r == hash.MD5 { + o.md5sum = &str + } else if r == hash.SHA1 { + o.sha1sum = &str + } + return str, nil +} + +var shellEscapeRegex = regexp.MustCompile(`[^A-Za-z0-9_.,:/@\n-]`) + +// Escape a string s.t. it cannot cause unintended behavior +// when sending it to a shell. +func shellEscape(str string) string { + safe := shellEscapeRegex.ReplaceAllString(str, `\$0`) + return strings.Replace(safe, "\n", "'\n'", -1) +} + +// Converts a byte array from the SSH session returned by +// an invocation of md5sum/sha1sum to a hash string +// as expected by the rest of this application +func parseHash(bytes []byte) string { + return strings.Split(string(bytes), " ")[0] // Split at hash / filename separator +} + +// Size returns the size in bytes of the remote sftp file +func (o *Object) Size() int64 { + return o.size +} + +// ModTime returns the modification time of the remote sftp file +func (o *Object) ModTime() time.Time { + return o.modTime +} + +// path returns the native path of the object +func (o *Object) path() string { + return path.Join(o.fs.root, o.remote) +} + +// setMetadata updates the info in the object from the stat result passed in +func (o *Object) setMetadata(info os.FileInfo) { + o.modTime = info.ModTime() + o.size = info.Size() + o.mode = info.Mode() +} + +// statRemote stats the file or directory at the remote given +func (f *Fs) stat(remote string) (info os.FileInfo, err error) { + c, err := f.getSftpConnection() + if err != nil { + return nil, errors.Wrap(err, "stat") + } + absPath := path.Join(f.root, remote) + info, err = c.sftpClient.Stat(absPath) + f.putSftpConnection(&c, err) + return info, err +} + +// stat updates the info in the Object +func (o *Object) stat() error { + info, err := o.fs.stat(o.remote) + if err != nil { + if os.IsNotExist(err) { + return fs.ErrorObjectNotFound + } + return errors.Wrap(err, "stat failed") + } + if info.IsDir() { + return errors.Wrapf(fs.ErrorNotAFile, "%q", o.remote) + } + o.setMetadata(info) + return nil +} + +// SetModTime sets the modification and access time to the specified time +// +// it also updates the info field +func (o *Object) SetModTime(modTime time.Time) error { + c, err := o.fs.getSftpConnection() + if err != nil { + return errors.Wrap(err, "SetModTime") + } + if o.fs.opt.SetModTime { + err = c.sftpClient.Chtimes(o.path(), modTime, modTime) + o.fs.putSftpConnection(&c, err) + if err != nil { + return errors.Wrap(err, "SetModTime failed") + } + } + err = o.stat() + if err != nil { + return errors.Wrap(err, "SetModTime stat failed") + } + return nil +} + +// Storable returns whether the remote sftp file is a regular file (not a directory, symbolic link, block device, character device, named pipe, etc) +func (o *Object) Storable() bool { + return o.mode.IsRegular() +} + +// objectReader represents a file open for reading on the SFTP server +type objectReader struct { + sftpFile *sftp.File + pipeReader *io.PipeReader + done chan struct{} +} + +func newObjectReader(sftpFile *sftp.File) *objectReader { + pipeReader, pipeWriter := io.Pipe() + file := &objectReader{ + sftpFile: sftpFile, + pipeReader: pipeReader, + done: make(chan struct{}), + } + + go func() { + // Use sftpFile.WriteTo to pump data so that it gets a + // chance to build the window up. + _, err := sftpFile.WriteTo(pipeWriter) + // Close the pipeWriter so the pipeReader fails with + // the same error or EOF if err == nil + _ = pipeWriter.CloseWithError(err) + // signal that we've finished + close(file.done) + }() + + return file +} + +// Read from a remote sftp file object reader +func (file *objectReader) Read(p []byte) (n int, err error) { + n, err = file.pipeReader.Read(p) + return n, err +} + +// Close a reader of a remote sftp file +func (file *objectReader) Close() (err error) { + // Close the sftpFile - this will likely cause the WriteTo to error + err = file.sftpFile.Close() + // Close the pipeReader so writes to the pipeWriter fail + _ = file.pipeReader.Close() + // Wait for the background process to finish + <-file.done + return err +} + +// Open a remote sftp file object for reading. Seek is supported +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + var offset, limit int64 = 0, -1 + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + case *fs.RangeOption: + offset, limit = x.Decode(o.Size()) + default: + if option.Mandatory() { + fs.Logf(o, "Unsupported mandatory option: %v", option) + } + } + } + c, err := o.fs.getSftpConnection() + if err != nil { + return nil, errors.Wrap(err, "Open") + } + sftpFile, err := c.sftpClient.Open(o.path()) + o.fs.putSftpConnection(&c, err) + if err != nil { + return nil, errors.Wrap(err, "Open failed") + } + if offset > 0 { + off, err := sftpFile.Seek(offset, io.SeekStart) + if err != nil || off != offset { + return nil, errors.Wrap(err, "Open Seek failed") + } + } + in = readers.NewLimitedReadCloser(newObjectReader(sftpFile), limit) + return in, nil +} + +// Update a remote sftp file using the data and ModTime from +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + // Clear the hash cache since we are about to update the object + o.md5sum = nil + o.sha1sum = nil + c, err := o.fs.getSftpConnection() + if err != nil { + return errors.Wrap(err, "Update") + } + file, err := c.sftpClient.Create(o.path()) + o.fs.putSftpConnection(&c, err) + if err != nil { + return errors.Wrap(err, "Update Create failed") + } + // remove the file if upload failed + remove := func() { + c, removeErr := o.fs.getSftpConnection() + if removeErr != nil { + fs.Debugf(src, "Failed to open new SSH connection for delete: %v", removeErr) + return + } + removeErr = c.sftpClient.Remove(o.path()) + o.fs.putSftpConnection(&c, removeErr) + if removeErr != nil { + fs.Debugf(src, "Failed to remove: %v", removeErr) + } else { + fs.Debugf(src, "Removed after failed upload: %v", err) + } + } + _, err = file.ReadFrom(in) + if err != nil { + remove() + return errors.Wrap(err, "Update ReadFrom failed") + } + err = file.Close() + if err != nil { + remove() + return errors.Wrap(err, "Update Close failed") + } + err = o.SetModTime(src.ModTime()) + if err != nil { + return errors.Wrap(err, "Update SetModTime failed") + } + return nil +} + +// Remove a remote sftp file object +func (o *Object) Remove() error { + c, err := o.fs.getSftpConnection() + if err != nil { + return errors.Wrap(err, "Remove") + } + err = c.sftpClient.Remove(o.path()) + o.fs.putSftpConnection(&c, err) + return err +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.Mover = &Fs{} + _ fs.DirMover = &Fs{} + _ fs.Object = &Object{} +) diff --git a/.rclone_repo/backend/sftp/sftp_internal_test.go b/.rclone_repo/backend/sftp/sftp_internal_test.go new file mode 100755 index 0000000..ad99ff7 --- /dev/null +++ b/.rclone_repo/backend/sftp/sftp_internal_test.go @@ -0,0 +1,37 @@ +// +build !plan9,go1.9 + +package sftp + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestShellEscape(t *testing.T) { + for i, test := range []struct { + unescaped, escaped string + }{ + {"", ""}, + {"/this/is/harmless", "/this/is/harmless"}, + {"$(rm -rf /)", "\\$\\(rm\\ -rf\\ /\\)"}, + {"/test/\n", "/test/'\n'"}, + {":\"'", ":\\\"\\'"}, + } { + got := shellEscape(test.unescaped) + assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped)) + } +} + +func TestParseHash(t *testing.T) { + for i, test := range []struct { + sshOutput, checksum string + }{ + {"8dbc7733dbd10d2efc5c0a0d8dad90f958581821 RELEASE.md\n", "8dbc7733dbd10d2efc5c0a0d8dad90f958581821"}, + {"03cfd743661f07975fa2f1220c5194cbaff48451 -\n", "03cfd743661f07975fa2f1220c5194cbaff48451"}, + } { + got := parseHash([]byte(test.sshOutput)) + assert.Equal(t, test.checksum, got, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput)) + } +} diff --git a/.rclone_repo/backend/sftp/sftp_test.go b/.rclone_repo/backend/sftp/sftp_test.go new file mode 100755 index 0000000..10d7376 --- /dev/null +++ b/.rclone_repo/backend/sftp/sftp_test.go @@ -0,0 +1,20 @@ +// Test Sftp filesystem interface + +// +build !plan9,go1.9 + +package sftp_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/sftp" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestSftp:", + NilObject: (*sftp.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/sftp/sftp_unsupported.go b/.rclone_repo/backend/sftp/sftp_unsupported.go new file mode 100755 index 0000000..e9ecd15 --- /dev/null +++ b/.rclone_repo/backend/sftp/sftp_unsupported.go @@ -0,0 +1,6 @@ +// Build for sftp for unsupported platforms to stop go complaining +// about "no buildable Go source files " + +// +build plan9 !go1.9 + +package sftp diff --git a/.rclone_repo/backend/sftp/stringlock.go b/.rclone_repo/backend/sftp/stringlock.go new file mode 100755 index 0000000..a1bfab4 --- /dev/null +++ b/.rclone_repo/backend/sftp/stringlock.go @@ -0,0 +1,49 @@ +// +build !plan9,go1.9 + +package sftp + +import "sync" + +// stringLock locks for string IDs passed in +type stringLock struct { + mu sync.Mutex // mutex to protect below + locks map[string]chan struct{} // map of locks +} + +// newStringLock creates a stringLock +func newStringLock() *stringLock { + return &stringLock{ + locks: make(map[string]chan struct{}), + } +} + +// Lock locks on the id passed in +func (l *stringLock) Lock(ID string) { + l.mu.Lock() + for { + ch, ok := l.locks[ID] + if !ok { + break + } + // Wait for the channel to be closed + l.mu.Unlock() + // fs.Logf(nil, "Waiting for stringLock on %q", ID) + <-ch + l.mu.Lock() + } + l.locks[ID] = make(chan struct{}) + l.mu.Unlock() +} + +// Unlock unlocks on the id passed in. Will panic if Lock with the +// given id wasn't called first. +func (l *stringLock) Unlock(ID string) { + l.mu.Lock() + ch, ok := l.locks[ID] + if !ok { + panic("stringLock: Unlock before Lock") + } + close(ch) + delete(l.locks, ID) + l.mu.Unlock() +} diff --git a/.rclone_repo/backend/sftp/stringlock_test.go b/.rclone_repo/backend/sftp/stringlock_test.go new file mode 100755 index 0000000..6e368f4 --- /dev/null +++ b/.rclone_repo/backend/sftp/stringlock_test.go @@ -0,0 +1,42 @@ +// +build !plan9,go1.9 + +package sftp + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestStringLock(t *testing.T) { + var wg sync.WaitGroup + counter := [3]int{} + lock := newStringLock() + const ( + outer = 10 + inner = 100 + total = outer * inner + ) + for k := 0; k < outer; k++ { + for j := range counter { + wg.Add(1) + go func(j int) { + defer wg.Done() + ID := fmt.Sprintf("%d", j) + for i := 0; i < inner; i++ { + lock.Lock(ID) + n := counter[j] + time.Sleep(1 * time.Millisecond) + counter[j] = n + 1 + lock.Unlock(ID) + } + + }(j) + } + } + wg.Wait() + assert.Equal(t, [3]int{total, total, total}, counter) +} diff --git a/.rclone_repo/backend/swift/auth.go b/.rclone_repo/backend/swift/auth.go new file mode 100755 index 0000000..b12cf4c --- /dev/null +++ b/.rclone_repo/backend/swift/auth.go @@ -0,0 +1,77 @@ +package swift + +import ( + "net/http" + + "github.com/ncw/swift" +) + +// auth is an authenticator for swift. It overrides the StorageUrl +// and AuthToken with fixed values. +type auth struct { + parentAuth swift.Authenticator + storageURL string + authToken string +} + +// newAuth creates a swift authenticator wrapper to override the +// StorageUrl and AuthToken values. +// +// Note that parentAuth can be nil +func newAuth(parentAuth swift.Authenticator, storageURL string, authToken string) *auth { + return &auth{ + parentAuth: parentAuth, + storageURL: storageURL, + authToken: authToken, + } +} + +// Request creates an http.Request for the auth - return nil if not needed +func (a *auth) Request(c *swift.Connection) (*http.Request, error) { + if a.parentAuth == nil { + return nil, nil + } + return a.parentAuth.Request(c) +} + +// Response parses the http.Response +func (a *auth) Response(resp *http.Response) error { + if a.parentAuth == nil { + return nil + } + return a.parentAuth.Response(resp) +} + +// The public storage URL - set Internal to true to read +// internal/service net URL +func (a *auth) StorageUrl(Internal bool) string { // nolint + if a.storageURL != "" { + return a.storageURL + } + if a.parentAuth == nil { + return "" + } + return a.parentAuth.StorageUrl(Internal) +} + +// The access token +func (a *auth) Token() string { + if a.authToken != "" { + return a.authToken + } + if a.parentAuth == nil { + return "" + } + return a.parentAuth.Token() +} + +// The CDN url if available +func (a *auth) CdnUrl() string { // nolint + if a.parentAuth == nil { + return "" + } + return a.parentAuth.CdnUrl() +} + +// Check the interfaces are satisfied +var _ swift.Authenticator = (*auth)(nil) diff --git a/.rclone_repo/backend/swift/swift.go b/.rclone_repo/backend/swift/swift.go new file mode 100755 index 0000000..a490b20 --- /dev/null +++ b/.rclone_repo/backend/swift/swift.go @@ -0,0 +1,1049 @@ +// Package swift provides an interface to the Swift object storage system +package swift + +import ( + "bufio" + "bytes" + "fmt" + "io" + "path" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/fs/operations" + "github.com/ncw/rclone/fs/walk" + "github.com/ncw/swift" + "github.com/pkg/errors" +) + +// Constants +const ( + directoryMarkerContentType = "application/directory" // content type of directory marker objects + listChunks = 1000 // chunk size to read directory listings +) + +// SharedOptions are shared between swift and hubic +var SharedOptions = []fs.Option{{ + Name: "chunk_size", + Help: "Above this size files will be chunked into a _segments container.", + Default: fs.SizeSuffix(5 * 1024 * 1024 * 1024), + Advanced: true, +}} + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "swift", + Description: "Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)", + NewFs: NewFs, + Options: append([]fs.Option{{ + Name: "env_auth", + Help: "Get swift credentials from environment variables in standard OpenStack form.", + Default: false, + Examples: []fs.OptionExample{ + { + Value: "false", + Help: "Enter swift credentials in the next step", + }, { + Value: "true", + Help: "Get swift credentials from environment vars. Leave other fields blank if using this.", + }, + }, + }, { + Name: "user", + Help: "User name to log in (OS_USERNAME).", + }, { + Name: "key", + Help: "API key or password (OS_PASSWORD).", + }, { + Name: "auth", + Help: "Authentication URL for server (OS_AUTH_URL).", + Examples: []fs.OptionExample{{ + Help: "Rackspace US", + Value: "https://auth.api.rackspacecloud.com/v1.0", + }, { + Help: "Rackspace UK", + Value: "https://lon.auth.api.rackspacecloud.com/v1.0", + }, { + Help: "Rackspace v2", + Value: "https://identity.api.rackspacecloud.com/v2.0", + }, { + Help: "Memset Memstore UK", + Value: "https://auth.storage.memset.com/v1.0", + }, { + Help: "Memset Memstore UK v2", + Value: "https://auth.storage.memset.com/v2.0", + }, { + Help: "OVH", + Value: "https://auth.cloud.ovh.net/v2.0", + }}, + }, { + Name: "user_id", + Help: "User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID).", + }, { + Name: "domain", + Help: "User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME)", + }, { + Name: "tenant", + Help: "Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME)", + }, { + Name: "tenant_id", + Help: "Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID)", + }, { + Name: "tenant_domain", + Help: "Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME)", + }, { + Name: "region", + Help: "Region name - optional (OS_REGION_NAME)", + }, { + Name: "storage_url", + Help: "Storage URL - optional (OS_STORAGE_URL)", + }, { + Name: "auth_token", + Help: "Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)", + }, { + Name: "auth_version", + Help: "AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)", + Default: 0, + }, { + Name: "endpoint_type", + Help: "Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE)", + Default: "public", + Examples: []fs.OptionExample{{ + Help: "Public (default, choose this if not sure)", + Value: "public", + }, { + Help: "Internal (use internal service net)", + Value: "internal", + }, { + Help: "Admin", + Value: "admin", + }}, + }, { + Name: "storage_policy", + Help: "The storage policy to use when creating a new container", + Default: "", + Examples: []fs.OptionExample{{ + Help: "Default", + Value: "", + }, { + Help: "OVH Public Cloud Storage", + Value: "pcs", + }, { + Help: "OVH Public Cloud Archive", + Value: "pca", + }}, + }}, SharedOptions...), + }) +} + +// Options defines the configuration for this backend +type Options struct { + EnvAuth bool `config:"env_auth"` + User string `config:"user"` + Key string `config:"key"` + Auth string `config:"auth"` + UserID string `config:"user_id"` + Domain string `config:"domain"` + Tenant string `config:"tenant"` + TenantID string `config:"tenant_id"` + TenantDomain string `config:"tenant_domain"` + Region string `config:"region"` + StorageURL string `config:"storage_url"` + AuthToken string `config:"auth_token"` + AuthVersion int `config:"auth_version"` + StoragePolicy string `config:"storage_policy"` + EndpointType string `config:"endpoint_type"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` +} + +// Fs represents a remote swift server +type Fs struct { + name string // name of this remote + root string // the path we are working on if any + features *fs.Features // optional features + opt Options // options for this backend + c *swift.Connection // the connection to the swift server + container string // the container we are working on + containerOKMu sync.Mutex // mutex to protect container OK + containerOK bool // true if we have created the container + segmentsContainer string // container to store the segments (if any) in + noCheckContainer bool // don't check the container before creating it +} + +// Object describes a swift object +// +// Will definitely have info but maybe not meta +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + info swift.Object // Info from the swift object if known + headers swift.Headers // The object headers if known +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + if f.root == "" { + return f.container + } + return f.container + "/" + f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + if f.root == "" { + return fmt.Sprintf("Swift container %s", f.container) + } + return fmt.Sprintf("Swift container %s path %s", f.container, f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Pattern to match a swift path +var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`) + +// parseParse parses a swift 'url' +func parsePath(path string) (container, directory string, err error) { + parts := matcher.FindStringSubmatch(path) + if parts == nil { + err = errors.Errorf("couldn't find container in swift path %q", path) + } else { + container, directory = parts[1], parts[2] + directory = strings.Trim(directory, "/") + } + return +} + +// swiftConnection makes a connection to swift +func swiftConnection(opt *Options, name string) (*swift.Connection, error) { + c := &swift.Connection{ + // Keep these in the same order as the Config for ease of checking + UserName: opt.User, + ApiKey: opt.Key, + AuthUrl: opt.Auth, + UserId: opt.UserID, + Domain: opt.Domain, + Tenant: opt.Tenant, + TenantId: opt.TenantID, + TenantDomain: opt.TenantDomain, + Region: opt.Region, + StorageUrl: opt.StorageURL, + AuthToken: opt.AuthToken, + AuthVersion: opt.AuthVersion, + EndpointType: swift.EndpointType(opt.EndpointType), + ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport + Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport + Transport: fshttp.NewTransport(fs.Config), + } + if opt.EnvAuth { + err := c.ApplyEnvironment() + if err != nil { + return nil, errors.Wrap(err, "failed to read environment variables") + } + } + StorageUrl, AuthToken := c.StorageUrl, c.AuthToken // nolint + if !c.Authenticated() { + if c.UserName == "" && c.UserId == "" { + return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)") + } + if c.ApiKey == "" { + return nil, errors.New("key not found") + } + if c.AuthUrl == "" { + return nil, errors.New("auth not found") + } + err := c.Authenticate() // fills in c.StorageUrl and c.AuthToken + if err != nil { + return nil, err + } + } + // Make sure we re-auth with the AuthToken and StorageUrl + // provided by wrapping the existing auth, so we can just + // override one or the other or both. + if StorageUrl != "" || AuthToken != "" { + // Re-write StorageURL and AuthToken if they are being + // overridden as c.Authenticate above will have + // overwritten them. + if StorageUrl != "" { + c.StorageUrl = StorageUrl + } + if AuthToken != "" { + c.AuthToken = AuthToken + } + c.Auth = newAuth(c.Auth, StorageUrl, AuthToken) + } + return c, nil +} + +// NewFsWithConnection constructs an Fs from the path, container:path +// and authenticated connection. +// +// if noCheckContainer is set then the Fs won't check the container +// exists before creating it. +func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, noCheckContainer bool) (fs.Fs, error) { + container, directory, err := parsePath(root) + if err != nil { + return nil, err + } + f := &Fs{ + name: name, + opt: *opt, + c: c, + container: container, + segmentsContainer: container + "_segments", + root: directory, + noCheckContainer: noCheckContainer, + } + f.features = (&fs.Features{ + ReadMimeType: true, + WriteMimeType: true, + BucketBased: true, + }).Fill(f) + if f.root != "" { + f.root += "/" + // Check to see if the object exists - ignoring directory markers + info, _, err := f.c.Object(container, directory) + if err == nil && info.ContentType != directoryMarkerContentType { + f.root = path.Dir(directory) + if f.root == "." { + f.root = "" + } else { + f.root += "/" + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + } + return f, nil +} + +// NewFs contstructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + c, err := swiftConnection(opt, name) + if err != nil { + return nil, err + } + return NewFsWithConnection(opt, name, root, c, false) +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *swift.Object) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + // Note that due to a quirk of swift, dynamic large objects are + // returned as 0 bytes in the listing. Correct this here by + // making sure we read the full metadata for all 0 byte files. + // We don't read the metadata for directory marker objects. + if info != nil && info.Bytes == 0 && info.ContentType != "application/directory" { + info = nil + } + if info != nil { + // Set info but not headers + o.info = *info + } else { + err := o.readMetaData() // reads info and headers, returning an error + if err != nil { + return nil, err + } + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found it +// returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// listFn is called from list and listContainerRoot to handle an object. +type listFn func(remote string, object *swift.Object, isDirectory bool) error + +// listContainerRoot lists the objects into the function supplied from +// the container and root supplied +// +// Set recurse to read sub directories +func (f *Fs) listContainerRoot(container, root string, dir string, recurse bool, fn listFn) error { + prefix := root + if dir != "" { + prefix += dir + "/" + } + // Options for ObjectsWalk + opts := swift.ObjectsOpts{ + Prefix: prefix, + Limit: listChunks, + } + if !recurse { + opts.Delimiter = '/' + } + rootLength := len(root) + return f.c.ObjectsWalk(container, &opts, func(opts *swift.ObjectsOpts) (interface{}, error) { + objects, err := f.c.Objects(container, opts) + if err == nil { + for i := range objects { + object := &objects[i] + isDirectory := false + if !recurse { + isDirectory = strings.HasSuffix(object.Name, "/") + } + if !strings.HasPrefix(object.Name, prefix) { + fs.Logf(f, "Odd name received %q", object.Name) + continue + } + if object.Name == prefix { + // If we have zero length directory markers ending in / then swift + // will return them in the listing for the directory which causes + // duplicate directories. Ignore them here. + continue + } + remote := object.Name[rootLength:] + err = fn(remote, object, isDirectory) + if err != nil { + break + } + } + } + return objects, err + }) +} + +type addEntryFn func(fs.DirEntry) error + +// list the objects into the function supplied +func (f *Fs) list(dir string, recurse bool, fn addEntryFn) error { + err := f.listContainerRoot(f.container, f.root, dir, recurse, func(remote string, object *swift.Object, isDirectory bool) (err error) { + if isDirectory { + remote = strings.TrimRight(remote, "/") + d := fs.NewDir(remote, time.Time{}).SetSize(object.Bytes) + err = fn(d) + } else { + // newObjectWithInfo does a full metadata read on 0 size objects which might be dynamic large objects + var o fs.Object + o, err = f.newObjectWithInfo(remote, object) + if err != nil { + return err + } + if o.Storable() { + err = fn(o) + } + } + return err + }) + if err == swift.ContainerNotFound { + err = fs.ErrorDirNotFound + } + return err +} + +// mark the container as being OK +func (f *Fs) markContainerOK() { + if f.container != "" { + f.containerOKMu.Lock() + f.containerOK = true + f.containerOKMu.Unlock() + } +} + +// listDir lists a single directory +func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { + if f.container == "" { + return nil, fs.ErrorListBucketRequired + } + // List the objects + err = f.list(dir, false, func(entry fs.DirEntry) error { + entries = append(entries, entry) + return nil + }) + if err != nil { + return nil, err + } + // container must be present if listing succeeded + f.markContainerOK() + return entries, nil +} + +// listContainers lists the containers +func (f *Fs) listContainers(dir string) (entries fs.DirEntries, err error) { + if dir != "" { + return nil, fs.ErrorListBucketRequired + } + containers, err := f.c.ContainersAll(nil) + if err != nil { + return nil, errors.Wrap(err, "container listing failed") + } + for _, container := range containers { + d := fs.NewDir(container.Name, time.Time{}).SetSize(container.Bytes).SetItems(container.Count) + entries = append(entries, d) + } + return entries, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + if f.container == "" { + return f.listContainers(dir) + } + return f.listDir(dir) +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + if f.container == "" { + return errors.New("container needed for recursive list") + } + list := walk.NewListRHelper(callback) + err = f.list(dir, true, func(entry fs.DirEntry) error { + return list.Add(entry) + }) + if err != nil { + return err + } + // container must be present if listing succeeded + f.markContainerOK() + return list.Flush() +} + +// About gets quota information +func (f *Fs) About() (*fs.Usage, error) { + containers, err := f.c.ContainersAll(nil) + if err != nil { + return nil, errors.Wrap(err, "container listing failed") + } + var total, objects int64 + for _, c := range containers { + total += c.Bytes + objects += c.Count + } + usage := &fs.Usage{ + Used: fs.NewUsageValue(total), // bytes in use + Objects: fs.NewUsageValue(objects), // objects in use + } + return usage, nil +} + +// Put the object into the container +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + // Temporary Object under construction + fs := &Object{ + fs: f, + remote: src.Remote(), + headers: swift.Headers{}, // Empty object headers to stop readMetaData being called + } + return fs, fs.Update(in, src, options...) +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + f.containerOKMu.Lock() + defer f.containerOKMu.Unlock() + if f.containerOK { + return nil + } + // if we are at the root, then it is OK + if f.container == "" { + return nil + } + // Check to see if container exists first + var err error = swift.ContainerNotFound + if !f.noCheckContainer { + _, _, err = f.c.Container(f.container) + } + if err == swift.ContainerNotFound { + headers := swift.Headers{} + if f.opt.StoragePolicy != "" { + headers["X-Storage-Policy"] = f.opt.StoragePolicy + } + err = f.c.ContainerCreate(f.container, headers) + } + if err == nil { + f.containerOK = true + } + return err +} + +// Rmdir deletes the container if the fs is at the root +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + f.containerOKMu.Lock() + defer f.containerOKMu.Unlock() + if f.root != "" || dir != "" { + return nil + } + err := f.c.ContainerDelete(f.container) + if err == nil { + f.containerOK = false + } + return err +} + +// Precision of the remote +func (f *Fs) Precision() time.Duration { + return time.Nanosecond +} + +// Purge deletes all the files and directories +// +// Implemented here so we can make sure we delete directory markers +func (f *Fs) Purge() error { + // Delete all the files including the directory markers + toBeDeleted := make(chan fs.Object, fs.Config.Transfers) + delErr := make(chan error, 1) + go func() { + delErr <- operations.DeleteFiles(toBeDeleted) + }() + err := f.list("", true, func(entry fs.DirEntry) error { + if o, ok := entry.(*Object); ok { + toBeDeleted <- o + } + return nil + }) + close(toBeDeleted) + delError := <-delErr + if err == nil { + err = delError + } + if err != nil { + return err + } + return f.Rmdir("") +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + err := f.Mkdir("") + if err != nil { + return nil, err + } + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + srcFs := srcObj.fs + _, err = f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil) + if err != nil { + return nil, err + } + return f.NewObject(remote) +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the Md5sum of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.MD5 { + return "", hash.ErrUnsupported + } + isDynamicLargeObject, err := o.isDynamicLargeObject() + if err != nil { + return "", err + } + isStaticLargeObject, err := o.isStaticLargeObject() + if err != nil { + return "", err + } + if isDynamicLargeObject || isStaticLargeObject { + fs.Debugf(o, "Returning empty Md5sum for swift large object") + return "", nil + } + return strings.ToLower(o.info.Hash), nil +} + +// hasHeader checks for the header passed in returning false if the +// object isn't found. +func (o *Object) hasHeader(header string) (bool, error) { + err := o.readMetaData() + if err != nil { + if err == fs.ErrorObjectNotFound { + return false, nil + } + return false, err + } + _, isDynamicLargeObject := o.headers[header] + return isDynamicLargeObject, nil +} + +// isDynamicLargeObject checks for X-Object-Manifest header +func (o *Object) isDynamicLargeObject() (bool, error) { + return o.hasHeader("X-Object-Manifest") +} + +// isStaticLargeObjectFile checks for the X-Static-Large-Object header +func (o *Object) isStaticLargeObject() (bool, error) { + return o.hasHeader("X-Static-Large-Object") +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.info.Bytes +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +// +// it returns fs.ErrorObjectNotFound if the object isn't found +func (o *Object) readMetaData() (err error) { + if o.headers != nil { + return nil + } + info, h, err := o.fs.c.Object(o.fs.container, o.fs.root+o.remote) + if err != nil { + if err == swift.ObjectNotFound { + return fs.ErrorObjectNotFound + } + return err + } + o.info = info + o.headers = h + return nil +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + if fs.Config.UseServerModTime { + return o.info.LastModified + } + err := o.readMetaData() + if err != nil { + fs.Debugf(o, "Failed to read metadata: %s", err) + return o.info.LastModified + } + modTime, err := o.headers.ObjectMetadata().GetModTime() + if err != nil { + // fs.Logf(o, "Failed to read mtime from object: %v", err) + return o.info.LastModified + } + return modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + err := o.readMetaData() + if err != nil { + return err + } + meta := o.headers.ObjectMetadata() + meta.SetModTime(modTime) + newHeaders := meta.ObjectHeaders() + for k, v := range newHeaders { + o.headers[k] = v + } + // Include any other metadata from request + for k, v := range o.headers { + if strings.HasPrefix(k, "X-Object-") { + newHeaders[k] = v + } + } + return o.fs.c.ObjectUpdate(o.fs.container, o.fs.root+o.remote, newHeaders) +} + +// Storable returns if this object is storable +// +// It compares the Content-Type to directoryMarkerContentType - that +// makes it a directory marker which is not storable. +func (o *Object) Storable() bool { + return o.info.ContentType != directoryMarkerContentType +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + headers := fs.OpenOptionHeaders(options) + _, isRanging := headers["Range"] + in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers) + return +} + +// min returns the smallest of x, y +func min(x, y int64) int64 { + if x < y { + return x + } + return y +} + +// removeSegments removes any old segments from o +// +// if except is passed in then segments with that prefix won't be deleted +func (o *Object) removeSegments(except string) error { + segmentsRoot := o.fs.root + o.remote + "/" + err := o.fs.listContainerRoot(o.fs.segmentsContainer, segmentsRoot, "", true, func(remote string, object *swift.Object, isDirectory bool) error { + if isDirectory { + return nil + } + if except != "" && strings.HasPrefix(remote, except) { + // fs.Debugf(o, "Ignoring current segment file %q in container %q", segmentsRoot+remote, o.fs.segmentsContainer) + return nil + } + segmentPath := segmentsRoot + remote + fs.Debugf(o, "Removing segment file %q in container %q", segmentPath, o.fs.segmentsContainer) + return o.fs.c.ObjectDelete(o.fs.segmentsContainer, segmentPath) + }) + if err != nil { + return err + } + // remove the segments container if empty, ignore errors + err = o.fs.c.ContainerDelete(o.fs.segmentsContainer) + if err == nil { + fs.Debugf(o, "Removed empty container %q", o.fs.segmentsContainer) + } + return nil +} + +// urlEncode encodes a string so that it is a valid URL +// +// We don't use any of Go's standard methods as we need `/` not +// encoded but we need '&' encoded. +func urlEncode(str string) string { + var buf bytes.Buffer + for i := 0; i < len(str); i++ { + c := str[i] + if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '/' || c == '.' { + _ = buf.WriteByte(c) + } else { + _, _ = buf.WriteString(fmt.Sprintf("%%%02X", c)) + } + } + return buf.String() +} + +// updateChunks updates the existing object using chunks to a separate +// container. It returns a string which prefixes current segments. +func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64, contentType string) (string, error) { + // Create the segmentsContainer if it doesn't exist + var err error + _, _, err = o.fs.c.Container(o.fs.segmentsContainer) + if err == swift.ContainerNotFound { + headers := swift.Headers{} + if o.fs.opt.StoragePolicy != "" { + headers["X-Storage-Policy"] = o.fs.opt.StoragePolicy + } + err = o.fs.c.ContainerCreate(o.fs.segmentsContainer, headers) + } + if err != nil { + return "", err + } + // Upload the chunks + left := size + i := 0 + uniquePrefix := fmt.Sprintf("%s/%d", swift.TimeToFloatString(time.Now()), size) + segmentsPath := fmt.Sprintf("%s%s/%s", o.fs.root, o.remote, uniquePrefix) + in := bufio.NewReader(in0) + for { + // can we read at least one byte? + if _, err := in.Peek(1); err != nil { + if left > 0 { + return "", err // read less than expected + } + fs.Debugf(o, "Uploading segments into %q seems done (%v)", o.fs.segmentsContainer, err) + break + } + n := int64(o.fs.opt.ChunkSize) + if size != -1 { + n = min(left, n) + headers["Content-Length"] = strconv.FormatInt(n, 10) // set Content-Length as we know it + left -= n + } + segmentReader := io.LimitReader(in, n) + segmentPath := fmt.Sprintf("%s/%08d", segmentsPath, i) + fs.Debugf(o, "Uploading segment file %q into %q", segmentPath, o.fs.segmentsContainer) + _, err := o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers) + if err != nil { + return "", err + } + i++ + } + // Upload the manifest + headers["X-Object-Manifest"] = urlEncode(fmt.Sprintf("%s/%s", o.fs.segmentsContainer, segmentsPath)) + headers["Content-Length"] = "0" // set Content-Length as we know it + emptyReader := bytes.NewReader(nil) + manifestName := o.fs.root + o.remote + _, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers) + return uniquePrefix + "/", err +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + if o.fs.container == "" { + return fserrors.FatalError(errors.New("container name needed in remote")) + } + err := o.fs.Mkdir("") + if err != nil { + return err + } + size := src.Size() + modTime := src.ModTime() + + // Note whether this is a dynamic large object before starting + isDynamicLargeObject, err := o.isDynamicLargeObject() + if err != nil { + return err + } + + // Set the mtime + m := swift.Metadata{} + m.SetModTime(modTime) + contentType := fs.MimeType(src) + headers := m.ObjectHeaders() + uniquePrefix := "" + if size > int64(o.fs.opt.ChunkSize) || size == -1 { + uniquePrefix, err = o.updateChunks(in, headers, size, contentType) + if err != nil { + return err + } + } else { + headers["Content-Length"] = strconv.FormatInt(size, 10) // set Content-Length as we know it + _, err := o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", contentType, headers) + if err != nil { + return err + } + } + + // If file was a dynamic large object then remove old/all segments + if isDynamicLargeObject { + err = o.removeSegments(uniquePrefix) + if err != nil { + fs.Logf(o, "Failed to remove old segments - carrying on with upload: %v", err) + } + } + + // Read the metadata from the newly created object + o.headers = nil // wipe old metadata + return o.readMetaData() +} + +// Remove an object +func (o *Object) Remove() error { + isDynamicLargeObject, err := o.isDynamicLargeObject() + if err != nil { + return err + } + // Remove file/manifest first + err = o.fs.c.ObjectDelete(o.fs.container, o.fs.root+o.remote) + if err != nil { + return err + } + // ...then segments if required + if isDynamicLargeObject { + err = o.removeSegments("") + if err != nil { + return err + } + } + return nil +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + return o.info.ContentType +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.Purger = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.Copier = &Fs{} + _ fs.ListRer = &Fs{} + _ fs.Object = &Object{} + _ fs.MimeTyper = &Object{} +) diff --git a/.rclone_repo/backend/swift/swift_internal_test.go b/.rclone_repo/backend/swift/swift_internal_test.go new file mode 100755 index 0000000..de95a4c --- /dev/null +++ b/.rclone_repo/backend/swift/swift_internal_test.go @@ -0,0 +1,25 @@ +package swift + +import "testing" + +func TestInternalUrlEncode(t *testing.T) { + for _, test := range []struct { + in string + want string + }{ + {"", ""}, + {"abcdefghijklmopqrstuvwxyz", "abcdefghijklmopqrstuvwxyz"}, + {"ABCDEFGHIJKLMOPQRSTUVWXYZ", "ABCDEFGHIJKLMOPQRSTUVWXYZ"}, + {"0123456789", "0123456789"}, + {"abc/ABC/123", "abc/ABC/123"}, + {" ", "%20%20%20"}, + {"&", "%26"}, + {"ߣ", "%C3%9F%C2%A3"}, + {"Vidéo Potato Sausage?&£.mkv", "Vid%C3%A9o%20Potato%20Sausage%3F%26%C2%A3.mkv"}, + } { + got := urlEncode(test.in) + if got != test.want { + t.Logf("%q: want %q got %q", test.in, test.want, got) + } + } +} diff --git a/.rclone_repo/backend/swift/swift_test.go b/.rclone_repo/backend/swift/swift_test.go new file mode 100755 index 0000000..eec0b99 --- /dev/null +++ b/.rclone_repo/backend/swift/swift_test.go @@ -0,0 +1,17 @@ +// Test Swift filesystem interface +package swift_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/swift" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestSwift:", + NilObject: (*swift.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/webdav/api/types.go b/.rclone_repo/backend/webdav/api/types.go new file mode 100755 index 0000000..b92cc97 --- /dev/null +++ b/.rclone_repo/backend/webdav/api/types.go @@ -0,0 +1,164 @@ +// Package api has type definitions for webdav +package api + +import ( + "encoding/xml" + "regexp" + "strconv" + "strings" + "time" +) + +const ( + // Wed, 27 Sep 2017 14:28:34 GMT + timeFormat = time.RFC1123 +) + +// Multistatus contains responses returned from an HTTP 207 return code +type Multistatus struct { + Responses []Response `xml:"response"` +} + +// Response contains an Href the response it about and its properties +type Response struct { + Href string `xml:"href"` + Props Prop `xml:"propstat"` +} + +// Prop is the properties of a response +// +// This is a lazy way of decoding the multiple in the +// response. +// +// The response might look like this +// +// +// /remote.php/webdav/Nextcloud%20Manual.pdf +// +// +// Tue, 19 Dec 2017 22:02:36 GMT +// 4143665 +// +// "048d7be4437ff7deeae94db50ff3e209" +// application/pdf +// +// HTTP/1.1 200 OK +// +// +// +// +// +// +// HTTP/1.1 404 Not Found +// +// +// +// So we elide the array of and within that the array of +// into one struct. +// +// Note that status collects all the status values for which we just +// check the first is OK. +type Prop struct { + Status []string `xml:"DAV: status"` + Name string `xml:"DAV: prop>displayname,omitempty"` + Type *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"` + Size int64 `xml:"DAV: prop>getcontentlength,omitempty"` + Modified Time `xml:"DAV: prop>getlastmodified,omitempty"` +} + +// Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200" +var parseStatus = regexp.MustCompile(`^HTTP/[0-9.]+\s+(\d+)`) + +// StatusOK examines the Status and returns an OK flag +func (p *Prop) StatusOK() bool { + // Assume OK if no statuses received + if len(p.Status) == 0 { + return true + } + match := parseStatus.FindStringSubmatch(p.Status[0]) + if len(match) < 2 { + return false + } + code, err := strconv.Atoi(match[1]) + if err != nil { + return false + } + if code >= 200 && code < 300 { + return true + } + return false +} + +// PropValue is a tagged name and value +type PropValue struct { + XMLName xml.Name `xml:""` + Value string `xml:",chardata"` +} + +// Error is used to desribe webdav errors +// +// +// Sabre\DAV\Exception\NotFound +// File with name Photo could not be located +// +type Error struct { + Exception string `xml:"exception,omitempty"` + Message string `xml:"message,omitempty"` + Status string + StatusCode int +} + +// Error returns a string for the error and statistifes the error interface +func (e *Error) Error() string { + var out []string + if e.Message != "" { + out = append(out, e.Message) + } + if e.Exception != "" { + out = append(out, e.Exception) + } + if e.Status != "" { + out = append(out, e.Status) + } + if len(out) == 0 { + return "Webdav Error" + } + return strings.Join(out, ": ") +} + +// Time represents represents date and time information for the +// webdav API marshalling to and from timeFormat +type Time time.Time + +// MarshalXML turns a Time into XML +func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + timeString := (*time.Time)(t).Format(timeFormat) + return e.EncodeElement(timeString, start) +} + +// Possible time formats to parse the time with +var timeFormats = []string{ + timeFormat, // Wed, 27 Sep 2017 14:28:34 GMT (as per RFC) + time.RFC1123Z, // Fri, 05 Jan 2018 14:14:38 +0000 (as used by mydrive.ch) + time.UnixDate, // Wed May 17 15:31:58 UTC 2017 (as used in an internal server) +} + +// UnmarshalXML turns XML into a Time +func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var v string + err := d.DecodeElement(&v, &start) + if err != nil { + return err + } + + // Parse the time format in multiple possible ways + var newT time.Time + for _, timeFormat := range timeFormats { + newT, err = time.Parse(timeFormat, v) + if err == nil { + *t = Time(newT) + break + } + } + return err +} diff --git a/.rclone_repo/backend/webdav/odrvcookie/fetch.go b/.rclone_repo/backend/webdav/odrvcookie/fetch.go new file mode 100755 index 0000000..2a7a3ee --- /dev/null +++ b/.rclone_repo/backend/webdav/odrvcookie/fetch.go @@ -0,0 +1,186 @@ +// Package odrvcookie can fetch authentication cookies for a sharepoint webdav endpoint +package odrvcookie + +import ( + "bytes" + "encoding/xml" + "html/template" + "net/http" + "net/http/cookiejar" + "net/url" + "strings" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/fshttp" + "github.com/pkg/errors" + "golang.org/x/net/publicsuffix" +) + +// CookieAuth hold the authentication information +// These are username and password as well as the authentication endpoint +type CookieAuth struct { + user string + pass string + endpoint string +} + +// CookieResponse contains the requested cookies +type CookieResponse struct { + RtFa http.Cookie + FedAuth http.Cookie +} + +// SuccessResponse hold a response from the sharepoint webdav +type SuccessResponse struct { + XMLName xml.Name `xml:"Envelope"` + Succ SuccessResponseBody `xml:"Body"` +} + +// SuccessResponseBody is the body of a success response, it holds the token +type SuccessResponseBody struct { + XMLName xml.Name + Type string `xml:"RequestSecurityTokenResponse>TokenType"` + Created time.Time `xml:"RequestSecurityTokenResponse>Lifetime>Created"` + Expires time.Time `xml:"RequestSecurityTokenResponse>Lifetime>Expires"` + Token string `xml:"RequestSecurityTokenResponse>RequestedSecurityToken>BinarySecurityToken"` +} + +// reqString is a template that gets populated with the user data in order to retrieve a "BinarySecurityToken" +const reqString = ` + +http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue + +http://www.w3.org/2005/08/addressing/anonymous + +https://login.microsoftonline.com/extSTS.srf + + + {{ .Username }} + {{ .Password }} + + + + + + + + {{ .Address }} + + +http://schemas.xmlsoap.org/ws/2005/05/identity/NoProofKey +http://schemas.xmlsoap.org/ws/2005/02/trust/Issue +urn:oasis:names:tc:SAML:1.0:assertion + + +` + +// New creates a new CookieAuth struct +func New(pUser, pPass, pEndpoint string) CookieAuth { + retStruct := CookieAuth{ + user: pUser, + pass: pPass, + endpoint: pEndpoint, + } + + return retStruct +} + +// Cookies creates a CookieResponse. It fetches the auth token and then +// retrieves the Cookies +func (ca *CookieAuth) Cookies() (*CookieResponse, error) { + tokenResp, err := ca.getSPToken() + if err != nil { + return nil, err + } + return ca.getSPCookie(tokenResp) +} + +func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error) { + spRoot, err := url.Parse(ca.endpoint) + if err != nil { + return nil, errors.Wrap(err, "Error while contructing endpoint URL") + } + + u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0") + if err != nil { + return nil, errors.Wrap(err, "Error while constructing login URL") + } + + // To authenticate with davfs or anything else we need two cookies (rtFa and FedAuth) + // In order to get them we use the token we got earlier and a cookieJar + jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + if err != nil { + return nil, err + } + + client := &http.Client{ + Jar: jar, + } + + // Send the previously aquired Token as a Post parameter + if _, err = client.Post(u.String(), "text/xml", strings.NewReader(conf.Succ.Token)); err != nil { + return nil, errors.Wrap(err, "Error while grabbing cookies from endpoint: %v") + } + + cookieResponse := CookieResponse{} + for _, cookie := range jar.Cookies(u) { + if (cookie.Name == "rtFa") || (cookie.Name == "FedAuth") { + switch cookie.Name { + case "rtFa": + cookieResponse.RtFa = *cookie + case "FedAuth": + cookieResponse.FedAuth = *cookie + } + } + } + return &cookieResponse, nil +} + +func (ca *CookieAuth) getSPToken() (conf *SuccessResponse, err error) { + reqData := map[string]interface{}{ + "Username": ca.user, + "Password": ca.pass, + "Address": ca.endpoint, + } + + t := template.Must(template.New("authXML").Parse(reqString)) + + buf := &bytes.Buffer{} + if err := t.Execute(buf, reqData); err != nil { + return nil, errors.Wrap(err, "Error while filling auth token template") + } + + // Create and execute the first request which returns an auth token for the sharepoint service + // With this token we can authenticate on the login page and save the returned cookies + req, err := http.NewRequest("POST", "https://login.microsoftonline.com/extSTS.srf", buf) + if err != nil { + return nil, err + } + + client := fshttp.NewClient(fs.Config) + resp, err := client.Do(req) + if err != nil { + return nil, errors.Wrap(err, "Error while logging in to endpoint") + } + defer fs.CheckClose(resp.Body, &err) + + respBuf := bytes.Buffer{} + _, err = respBuf.ReadFrom(resp.Body) + if err != nil { + return nil, err + } + s := respBuf.Bytes() + + conf = &SuccessResponse{} + err = xml.Unmarshal(s, conf) + if err != nil { + // FIXME: Try to parse with FailedResponse struct (check for server error code) + return nil, errors.Wrap(err, "Error while reading endpoint response") + } + + return +} diff --git a/.rclone_repo/backend/webdav/webdav.go b/.rclone_repo/backend/webdav/webdav.go new file mode 100755 index 0000000..25721a9 --- /dev/null +++ b/.rclone_repo/backend/webdav/webdav.go @@ -0,0 +1,1002 @@ +// Package webdav provides an interface to the Webdav +// object storage system. +package webdav + +// Owncloud: Getting Oc-Checksum: +// SHA1:f572d396fae9206628714fb2ce00f72e94f2258f on HEAD but not on +// nextcloud? + +// docs for file webdav +// https://docs.nextcloud.com/server/12/developer_manual/client_apis/WebDAV/index.html + +// indicates checksums can be set as metadata here +// https://github.com/nextcloud/server/issues/6129 +// owncloud seems to have checksums as metadata though - can read them + +// SetModTime might be possible +// https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file +// ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option. +// For example the ownCloud WebDAV server does it that way. + +import ( + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/ncw/rclone/backend/webdav/api" + "github.com/ncw/rclone/backend/webdav/odrvcookie" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/pacer" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" +) + +const ( + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential + defaultDepth = "1" // depth for PROPFIND +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "webdav", + Description: "Webdav", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "url", + Help: "URL of http host to connect to", + Required: true, + Examples: []fs.OptionExample{{ + Value: "https://example.com", + Help: "Connect to example.com", + }}, + }, { + Name: "vendor", + Help: "Name of the Webdav site/service/software you are using", + Examples: []fs.OptionExample{{ + Value: "nextcloud", + Help: "Nextcloud", + }, { + Value: "owncloud", + Help: "Owncloud", + }, { + Value: "sharepoint", + Help: "Sharepoint", + }, { + Value: "other", + Help: "Other site/service or software", + }}, + }, { + Name: "user", + Help: "User name", + }, { + Name: "pass", + Help: "Password.", + IsPassword: true, + }, { + Name: "bearer_token", + Help: "Bearer token instead of user/pass (eg a Macaroon)", + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + URL string `config:"url"` + Vendor string `config:"vendor"` + User string `config:"user"` + Pass string `config:"pass"` +} + +// Fs represents a remote webdav +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + features *fs.Features // optional features + endpoint *url.URL // URL of the host + endpointURL string // endpoint as a string + srv *rest.Client // the connection to the one drive server + pacer *pacer.Pacer // pacer for API calls + precision time.Duration // mod time precision + canStream bool // set if can stream + useOCMtime bool // set if can use X-OC-Mtime + retryWithZeroDepth bool // some vendors (sharepoint) won't list files when Depth is 1 (our default) +} + +// Object describes a webdav object +// +// Will definitely have info but maybe not meta +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + hasMetaData bool // whether info below has been set + size int64 // size of the object + modTime time.Time // modification time of the object + sha1 string // SHA-1 of the object content +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("webdav root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(resp *http.Response, err error) (bool, error) { + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// itemIsDir returns true if the item is a directory +// +// When a client sees a resourcetype it doesn't recognize it should +// assume it is a regular non-collection resource. [WebDav book by +// Lisa Dusseault ch 7.5.8 p170] +func itemIsDir(item *api.Response) bool { + if t := item.Props.Type; t != nil { + if t.Space == "DAV:" && t.Local == "collection" { + return true + } + fs.Debugf(nil, "Unknown resource type %q/%q on %q", t.Space, t.Local, item.Props.Name) + } + return false +} + +// readMetaDataForPath reads the metadata from the path +func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err error) { + // FIXME how do we read back additional properties? + opts := rest.Opts{ + Method: "PROPFIND", + Path: f.filePath(path), + ExtraHeaders: map[string]string{ + "Depth": depth, + }, + NoRedirect: true, + } + var result api.Multistatus + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(&opts, nil, &result) + return shouldRetry(resp, err) + }) + if apiErr, ok := err.(*api.Error); ok { + // does not exist + switch apiErr.StatusCode { + case http.StatusNotFound: + if f.retryWithZeroDepth && depth != "0" { + return f.readMetaDataForPath(path, "0") + } + return nil, fs.ErrorObjectNotFound + case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther: + // Some sort of redirect - go doesn't deal with these properly (it resets + // the method to GET). However we can assume that if it was redirected the + // object was not found. + return nil, fs.ErrorObjectNotFound + } + } + if err != nil { + return nil, errors.Wrap(err, "read metadata failed") + } + if len(result.Responses) < 1 { + return nil, fs.ErrorObjectNotFound + } + item := result.Responses[0] + if !item.Props.StatusOK() { + return nil, fs.ErrorObjectNotFound + } + if itemIsDir(&item) { + return nil, fs.ErrorNotAFile + } + return &item.Props, nil +} + +// errorHandler parses a non 2xx error response into an error +func errorHandler(resp *http.Response) error { + body, err := rest.ReadBody(resp) + if err != nil { + return errors.Wrap(err, "error when trying to read error from body") + } + // Decode error response + errResponse := new(api.Error) + err = xml.Unmarshal(body, &errResponse) + if err != nil { + // set the Message to be the body if can't parse the XML + errResponse.Message = strings.TrimSpace(string(body)) + } + errResponse.Status = resp.Status + errResponse.StatusCode = resp.StatusCode + return errResponse +} + +// addShlash makes sure s is terminated with a / if non empty +func addSlash(s string) string { + if s != "" && !strings.HasSuffix(s, "/") { + s += "/" + } + return s +} + +// filePath returns a file path (f.root, file) +func (f *Fs) filePath(file string) string { + return rest.URLPathEscape(path.Join(f.root, file)) +} + +// dirPath returns a directory path (f.root, dir) +func (f *Fs) dirPath(dir string) string { + return addSlash(f.filePath(dir)) +} + +// filePath returns a file path (f.root, remote) +func (o *Object) filePath() string { + return o.fs.filePath(o.remote) +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + rootIsDir := strings.HasSuffix(root, "/") + root = strings.Trim(root, "/") + + user := config.FileGet(name, "user") + pass := config.FileGet(name, "pass") + bearerToken := config.FileGet(name, "bearer_token") + if !strings.HasSuffix(opt.URL, "/") { + opt.URL += "/" + } + if opt.Pass != "" { + var err error + opt.Pass, err = obscure.Reveal(opt.Pass) + if err != nil { + return nil, errors.Wrap(err, "couldn't decrypt password") + } + } + if opt.Vendor == "" { + opt.Vendor = "other" + } + root = strings.Trim(root, "/") + + // Parse the endpoint + u, err := url.Parse(opt.URL) + if err != nil { + return nil, err + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + endpoint: u, + endpointURL: u.String(), + srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()), + pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), + precision: fs.ModTimeNotSupported, + } + f.features = (&fs.Features{ + CanHaveEmptyDirectories: true, + }).Fill(f) + if user != "" || pass != "" { + f.srv.SetUserPass(opt.User, opt.Pass) + } else if bearerToken != "" { + f.srv.SetHeader("Authorization", "BEARER "+bearerToken) + } + f.srv.SetErrorHandler(errorHandler) + err = f.setQuirks(opt.Vendor) + if err != nil { + return nil, err + } + + if root != "" && !rootIsDir { + // Check to see if the root actually an existing file + remote := path.Base(root) + f.root = path.Dir(root) + if f.root == "." { + f.root = "" + } + _, err := f.NewObject(remote) + if err != nil { + if errors.Cause(err) == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile { + // File doesn't exist so return old f + f.root = root + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + return f, nil +} + +// setQuirks adjusts the Fs for the vendor passed in +func (f *Fs) setQuirks(vendor string) error { + switch vendor { + case "owncloud": + f.canStream = true + f.precision = time.Second + f.useOCMtime = true + case "nextcloud": + f.precision = time.Second + f.useOCMtime = true + case "sharepoint": + // To mount sharepoint, two Cookies are required + // They have to be set instead of BasicAuth + f.srv.RemoveHeader("Authorization") // We don't need this Header if using cookies + spCk := odrvcookie.New(f.opt.User, f.opt.Pass, f.endpointURL) + spCookies, err := spCk.Cookies() + if err != nil { + return err + } + f.srv.SetCookie(&spCookies.FedAuth, &spCookies.RtFa) + + // sharepoint, unlike the other vendors, only lists files if the depth header is set to 0 + // however, rclone defaults to 1 since it provides recursive directory listing + // to determine if we may have found a file, the request has to be resent + // with the depth set to 0 + f.retryWithZeroDepth = true + case "other": + default: + fs.Debugf(f, "Unknown vendor %q", vendor) + } + + // Remove PutStream from optional features + if !f.canStream { + f.features.PutStream = nil + } + return nil +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *api.Prop) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + // Set info + err = o.setMetaData(info) + } else { + err = o.readMetaData() // reads info and meta, returning an error + } + if err != nil { + return nil, err + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// list the objects into the function supplied +// +// If directories is set it only sends directories +// User function to process a File item from listAll +// +// Should return true to finish processing +type listAllFn func(string, bool, *api.Prop) bool + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth string, fn listAllFn) (found bool, err error) { + opts := rest.Opts{ + Method: "PROPFIND", + Path: f.dirPath(dir), // FIXME Should not start with / + ExtraHeaders: map[string]string{ + "Depth": depth, + }, + } + var result api.Multistatus + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(&opts, nil, &result) + return shouldRetry(resp, err) + }) + if err != nil { + if apiErr, ok := err.(*api.Error); ok { + // does not exist + if apiErr.StatusCode == http.StatusNotFound { + if f.retryWithZeroDepth && depth != "0" { + return f.listAll(dir, directoriesOnly, filesOnly, "0", fn) + } + return found, fs.ErrorDirNotFound + } + } + return found, errors.Wrap(err, "couldn't list files") + } + //fmt.Printf("result = %#v", &result) + baseURL, err := rest.URLJoin(f.endpoint, opts.Path) + if err != nil { + return false, errors.Wrap(err, "couldn't join URL") + } + for i := range result.Responses { + item := &result.Responses[i] + isDir := itemIsDir(item) + + // Find name + u, err := rest.URLJoin(baseURL, item.Href) + if err != nil { + fs.Errorf(nil, "URL Join failed for %q and %q: %v", baseURL, item.Href, err) + continue + } + // Make sure directories end with a / + if isDir { + u.Path = addSlash(u.Path) + } + if !strings.HasPrefix(u.Path, baseURL.Path) { + fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, baseURL.Path) + continue + } + remote := path.Join(dir, u.Path[len(baseURL.Path):]) + if strings.HasSuffix(remote, "/") { + remote = remote[:len(remote)-1] + } + + // the listing contains info about itself which we ignore + if remote == dir { + continue + } + + // Check OK + if !item.Props.StatusOK() { + fs.Debugf(remote, "Ignoring item with bad status %q", item.Props.Status) + continue + } + + if isDir { + if filesOnly { + continue + } + } else { + if directoriesOnly { + continue + } + } + // item.Name = restoreReservedChars(item.Name) + if fn(remote, isDir, &item.Props) { + found = true + break + } + } + return +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + var iErr error + _, err = f.listAll(dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool { + if isDir { + d := fs.NewDir(remote, time.Time(info.Modified)) + // .SetID(info.ID) + // FIXME more info from dir? can set size, items? + entries = append(entries, d) + } else { + o, err := f.newObjectWithInfo(remote, info) + if err != nil { + iErr = err + return true + } + entries = append(entries, o) + } + return false + }) + if err != nil { + return nil, err + } + if iErr != nil { + return nil, iErr + } + return entries, nil +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Used to create new objects +func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object) { + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + size: size, + modTime: modTime, + } + return o +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + o := f.createObject(src.Remote(), src.ModTime(), src.Size()) + return o, o.Update(in, src, options...) +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// mkParentDir makes the parent of the native path dirPath if +// necessary and any directories above that +func (f *Fs) mkParentDir(dirPath string) error { + // defer log.Trace(dirPath, "")("") + // chop off trailing / if it exists + if strings.HasSuffix(dirPath, "/") { + dirPath = dirPath[:len(dirPath)-1] + } + parent := path.Dir(dirPath) + if parent == "." { + parent = "" + } + return f.mkdir(parent) +} + +// mkdir makes the directory and parents using native paths +func (f *Fs) mkdir(dirPath string) error { + // defer log.Trace(dirPath, "")("") + // We assume the root is already ceated + if dirPath == "" { + return nil + } + // Collections must end with / + if !strings.HasSuffix(dirPath, "/") { + dirPath += "/" + } + opts := rest.Opts{ + Method: "MKCOL", + Path: dirPath, + NoResponse: true, + } + err := f.pacer.Call(func() (bool, error) { + resp, err := f.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if apiErr, ok := err.(*api.Error); ok { + // already exists + if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable { + return nil + } + // parent does not exists + if apiErr.StatusCode == http.StatusConflict { + err = f.mkParentDir(dirPath) + if err == nil { + err = f.mkdir(dirPath) + } + } + } + return err +} + +// Mkdir creates the directory if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + dirPath := f.dirPath(dir) + return f.mkdir(dirPath) +} + +// dirNotEmpty returns true if the directory exists and is not Empty +// +// if the directory does not exist then err will be ErrorDirNotFound +func (f *Fs) dirNotEmpty(dir string) (found bool, err error) { + return f.listAll(dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool { + return true + }) +} + +// purgeCheck removes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(dir string, check bool) error { + if check { + notEmpty, err := f.dirNotEmpty(dir) + if err != nil { + return err + } + if notEmpty { + return fs.ErrorDirectoryNotEmpty + } + } + opts := rest.Opts{ + Method: "DELETE", + Path: f.dirPath(dir), + NoResponse: true, + } + var resp *http.Response + var err error + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(&opts, nil, nil) + return shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "rmdir failed") + } + // FIXME parse Multistatus response + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + return f.purgeCheck(dir, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return f.precision +} + +// Copy or Move src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy/fs.ErrorCantMove +func (f *Fs) copyOrMove(src fs.Object, remote string, method string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + if method == "COPY" { + return nil, fs.ErrorCantCopy + } + return nil, fs.ErrorCantMove + } + dstPath := f.filePath(remote) + err := f.mkParentDir(dstPath) + if err != nil { + return nil, errors.Wrap(err, "Copy mkParentDir failed") + } + destinationURL, err := rest.URLJoin(f.endpoint, dstPath) + if err != nil { + return nil, errors.Wrap(err, "copyOrMove couldn't join URL") + } + var resp *http.Response + opts := rest.Opts{ + Method: method, + Path: srcObj.filePath(), + NoResponse: true, + ExtraHeaders: map[string]string{ + "Destination": destinationURL.String(), + "Overwrite": "F", + }, + } + if f.useOCMtime { + opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9) + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "Copy call failed") + } + dstObj, err := f.NewObject(remote) + if err != nil { + return nil, errors.Wrap(err, "Copy NewObject failed") + } + return dstObj, nil +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + return f.copyOrMove(src, remote, "COPY") +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + return f.purgeCheck("", false) +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + return f.copyOrMove(src, remote, "MOVE") +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := srcFs.filePath(srcRemote) + dstPath := f.filePath(dstRemote) + + // Check if destination exists + _, err := f.dirNotEmpty(dstRemote) + if err == nil { + return fs.ErrorDirExists + } + if err != fs.ErrorDirNotFound { + return errors.Wrap(err, "DirMove dirExists dst failed") + } + + // Make sure the parent directory exists + err = f.mkParentDir(dstPath) + if err != nil { + return errors.Wrap(err, "DirMove mkParentDir dst failed") + } + + destinationURL, err := rest.URLJoin(f.endpoint, dstPath) + if err != nil { + return errors.Wrap(err, "DirMove couldn't join URL") + } + + var resp *http.Response + opts := rest.Opts{ + Method: "MOVE", + Path: addSlash(srcPath), + NoResponse: true, + ExtraHeaders: map[string]string{ + "Destination": addSlash(destinationURL.String()), + "Overwrite": "F", + }, + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "DirMove MOVE call failed") + } + return nil +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.None) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the SHA-1 of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.SHA1 { + return "", hash.ErrUnsupported + } + return o.sha1, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return 0 + } + return o.size +} + +// setMetaData sets the metadata from info +func (o *Object) setMetaData(info *api.Prop) (err error) { + o.hasMetaData = true + o.size = info.Size + o.modTime = time.Time(info.Modified) + return nil +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData() (err error) { + if o.hasMetaData { + return nil + } + info, err := o.fs.readMetaDataForPath(o.remote, defaultDepth) + if err != nil { + return err + } + return o.setMetaData(info) +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) error { + return fs.ErrorCantSetModTime +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + Path: o.filePath(), + Options: options, + } + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + return resp.Body, err +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// If existing is set then it updates the object rather than creating a new one +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + err = o.fs.mkParentDir(o.filePath()) + if err != nil { + return errors.Wrap(err, "Update mkParentDir failed") + } + + size := src.Size() + var resp *http.Response + opts := rest.Opts{ + Method: "PUT", + Path: o.filePath(), + Body: in, + NoResponse: true, + ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365 + } + if o.fs.useOCMtime { + opts.ExtraHeaders = map[string]string{ + "X-OC-Mtime": fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9), + } + } + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if err != nil { + // Remove failed upload + _ = o.Remove() + return err + } + // read metadata from remote + o.hasMetaData = false + return o.readMetaData() +} + +// Remove an object +func (o *Object) Remove() error { + opts := rest.Opts{ + Method: "DELETE", + Path: o.filePath(), + NoResponse: true, + } + return o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.Call(&opts) + return shouldRetry(resp, err) + }) +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.PutStreamer = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.Object = (*Object)(nil) +) diff --git a/.rclone_repo/backend/webdav/webdav_test.go b/.rclone_repo/backend/webdav/webdav_test.go new file mode 100755 index 0000000..a4264fa --- /dev/null +++ b/.rclone_repo/backend/webdav/webdav_test.go @@ -0,0 +1,17 @@ +// Test Webdav filesystem interface +package webdav_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/webdav" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestWebdav:", + NilObject: (*webdav.Object)(nil), + }) +} diff --git a/.rclone_repo/backend/yandex/api/api_upload.go b/.rclone_repo/backend/yandex/api/api_upload.go new file mode 100755 index 0000000..8db4413 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/api_upload.go @@ -0,0 +1,34 @@ +package src + +//from yadisk + +import ( + "io" + "net/http" +) + +//RootAddr is the base URL for Yandex Disk API. +const RootAddr = "https://cloud-api.yandex.com" //also https://cloud-api.yandex.net and https://cloud-api.yandex.ru + +func (c *Client) setRequestScope(req *http.Request) { + req.Header.Add("Accept", "application/json") + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", "OAuth "+c.token) +} + +func (c *Client) scopedRequest(method, urlPath string, body io.Reader) (*http.Request, error) { + fullURL := RootAddr + if urlPath[:1] != "/" { + fullURL += "/" + urlPath + } else { + fullURL += urlPath + } + + req, err := http.NewRequest(method, fullURL, body) + if err != nil { + return req, err + } + + c.setRequestScope(req) + return req, nil +} diff --git a/.rclone_repo/backend/yandex/api/client.go b/.rclone_repo/backend/yandex/api/client.go new file mode 100755 index 0000000..07b7a34 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/client.go @@ -0,0 +1,135 @@ +package src + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/pkg/errors" +) + +//Client struct +type Client struct { + token string + basePath string + HTTPClient *http.Client +} + +//NewClient creates new client +func NewClient(token string, client ...*http.Client) *Client { + return newClientInternal( + token, + "https://cloud-api.yandex.com/v1/disk", //also "https://cloud-api.yandex.net/v1/disk" "https://cloud-api.yandex.ru/v1/disk" + client...) +} + +func newClientInternal(token string, basePath string, client ...*http.Client) *Client { + c := &Client{ + token: token, + basePath: basePath, + } + if len(client) != 0 { + c.HTTPClient = client[0] + } else { + c.HTTPClient = http.DefaultClient + } + return c +} + +//ErrorHandler type +type ErrorHandler func(*http.Response) error + +var defaultErrorHandler ErrorHandler = func(resp *http.Response) error { + if resp.StatusCode/100 == 5 { + return errors.New("server error") + } + + if resp.StatusCode/100 == 4 { + var response DiskClientError + contents, _ := ioutil.ReadAll(resp.Body) + err := json.Unmarshal(contents, &response) + if err != nil { + return err + } + return response + } + + if resp.StatusCode/100 == 3 { + return errors.New("redirect error") + } + return nil +} + +func (HTTPRequest *HTTPRequest) run(client *Client) ([]byte, error) { + var err error + values := make(url.Values) + if HTTPRequest.Parameters != nil { + for k, v := range HTTPRequest.Parameters { + values.Set(k, fmt.Sprintf("%v", v)) + } + } + + var req *http.Request + if HTTPRequest.Method == "POST" { + // TODO json serialize + req, err = http.NewRequest( + "POST", + client.basePath+HTTPRequest.Path, + strings.NewReader(values.Encode())) + if err != nil { + return nil, err + } + // TODO + // req.Header.Set("Content-Type", "application/json") + } else { + req, err = http.NewRequest( + HTTPRequest.Method, + client.basePath+HTTPRequest.Path+"?"+values.Encode(), + nil) + if err != nil { + return nil, err + } + } + + for headerName := range HTTPRequest.Headers { + var headerValues = HTTPRequest.Headers[headerName] + for _, headerValue := range headerValues { + req.Header.Set(headerName, headerValue) + } + } + return runRequest(client, req) +} + +func runRequest(client *Client, req *http.Request) ([]byte, error) { + return runRequestWithErrorHandler(client, req, defaultErrorHandler) +} + +func runRequestWithErrorHandler(client *Client, req *http.Request, errorHandler ErrorHandler) (out []byte, err error) { + resp, err := client.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer CheckClose(resp.Body, &err) + + return checkResponseForErrorsWithErrorHandler(resp, errorHandler) +} + +func checkResponseForErrorsWithErrorHandler(resp *http.Response, errorHandler ErrorHandler) ([]byte, error) { + if resp.StatusCode/100 > 2 { + return nil, errorHandler(resp) + } + return ioutil.ReadAll(resp.Body) +} + +// CheckClose is a utility function used to check the return from +// Close in a defer statement. +func CheckClose(c io.Closer, err *error) { + cerr := c.Close() + if *err == nil { + *err = cerr + } +} diff --git a/.rclone_repo/backend/yandex/api/custom_property.go b/.rclone_repo/backend/yandex/api/custom_property.go new file mode 100755 index 0000000..9bddf20 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/custom_property.go @@ -0,0 +1,51 @@ +package src + +import ( + "bytes" + "encoding/json" + "io" + "net/url" +) + +//CustomPropertyResponse struct we send and is returned by the API for CustomProperty request. +type CustomPropertyResponse struct { + CustomProperties map[string]interface{} `json:"custom_properties"` +} + +//SetCustomProperty will set specified data from Yandex Disk +func (c *Client) SetCustomProperty(remotePath string, property string, value string) error { + rcm := map[string]interface{}{ + property: value, + } + cpr := CustomPropertyResponse{rcm} + data, _ := json.Marshal(cpr) + body := bytes.NewReader(data) + err := c.SetCustomPropertyRequest(remotePath, body) + if err != nil { + return err + } + return err +} + +//SetCustomPropertyRequest will make an CustomProperty request and return a URL to CustomProperty data to. +func (c *Client) SetCustomPropertyRequest(remotePath string, body io.Reader) (err error) { + values := url.Values{} + values.Add("path", remotePath) + req, err := c.scopedRequest("PATCH", "/v1/disk/resources?"+values.Encode(), body) + if err != nil { + return err + } + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + if err := CheckAPIError(resp); err != nil { + return err + } + defer CheckClose(resp.Body, &err) + + //If needed we can read response and check if custom_property is set. + + return nil +} diff --git a/.rclone_repo/backend/yandex/api/delete.go b/.rclone_repo/backend/yandex/api/delete.go new file mode 100755 index 0000000..c834411 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/delete.go @@ -0,0 +1,23 @@ +package src + +import ( + "net/url" + "strconv" +) + +// Delete will remove specified file/folder from Yandex Disk +func (c *Client) Delete(remotePath string, permanently bool) error { + + values := url.Values{} + values.Add("permanently", strconv.FormatBool(permanently)) + values.Add("path", remotePath) + urlPath := "/v1/disk/resources?" + values.Encode() + fullURL := RootAddr + if urlPath[:1] != "/" { + fullURL += "/" + urlPath + } else { + fullURL += urlPath + } + + return c.PerformDelete(fullURL) +} diff --git a/.rclone_repo/backend/yandex/api/disk_info_request.go b/.rclone_repo/backend/yandex/api/disk_info_request.go new file mode 100755 index 0000000..c615183 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/disk_info_request.go @@ -0,0 +1,48 @@ +package src + +import "encoding/json" + +//DiskInfoRequest type +type DiskInfoRequest struct { + client *Client + HTTPRequest *HTTPRequest +} + +func (req *DiskInfoRequest) request() *HTTPRequest { + return req.HTTPRequest +} + +//DiskInfoResponse struct is returned by the API for DiskInfo request. +type DiskInfoResponse struct { + TrashSize uint64 `json:"TrashSize"` + TotalSpace uint64 `json:"TotalSpace"` + UsedSpace uint64 `json:"UsedSpace"` + SystemFolders map[string]string `json:"SystemFolders"` +} + +//NewDiskInfoRequest create new DiskInfo Request +func (c *Client) NewDiskInfoRequest() *DiskInfoRequest { + return &DiskInfoRequest{ + client: c, + HTTPRequest: createGetRequest(c, "/", nil), + } +} + +//Exec run DiskInfo Request +func (req *DiskInfoRequest) Exec() (*DiskInfoResponse, error) { + data, err := req.request().run(req.client) + if err != nil { + return nil, err + } + + var info DiskInfoResponse + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + if info.SystemFolders == nil { + info.SystemFolders = make(map[string]string) + } + + return &info, nil +} diff --git a/.rclone_repo/backend/yandex/api/download.go b/.rclone_repo/backend/yandex/api/download.go new file mode 100755 index 0000000..310d346 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/download.go @@ -0,0 +1,66 @@ +package src + +import ( + "encoding/json" + "io" + "net/url" +) + +// DownloadResponse struct is returned by the API for Download request. +type DownloadResponse struct { + HRef string `json:"href"` + Method string `json:"method"` + Templated bool `json:"templated"` +} + +// Download will get specified data from Yandex.Disk supplying the extra headers +func (c *Client) Download(remotePath string, headers map[string]string) (io.ReadCloser, error) { //io.Writer + ur, err := c.DownloadRequest(remotePath) + if err != nil { + return nil, err + } + return c.PerformDownload(ur.HRef, headers) +} + +// DownloadRequest will make an download request and return a URL to download data to. +func (c *Client) DownloadRequest(remotePath string) (ur *DownloadResponse, err error) { + values := url.Values{} + values.Add("path", remotePath) + + req, err := c.scopedRequest("GET", "/v1/disk/resources/download?"+values.Encode(), nil) + if err != nil { + return nil, err + } + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + if err := CheckAPIError(resp); err != nil { + return nil, err + } + defer CheckClose(resp.Body, &err) + + ur, err = ParseDownloadResponse(resp.Body) + if err != nil { + return nil, err + } + + return ur, nil +} + +// ParseDownloadResponse tries to read and parse DownloadResponse struct. +func ParseDownloadResponse(data io.Reader) (*DownloadResponse, error) { + dec := json.NewDecoder(data) + var ur DownloadResponse + + if err := dec.Decode(&ur); err == io.EOF { + // ok + } else if err != nil { + return nil, err + } + + // TODO: check if there is any trash data after JSON and crash if there is. + + return &ur, nil +} diff --git a/.rclone_repo/backend/yandex/api/empty_trash.go b/.rclone_repo/backend/yandex/api/empty_trash.go new file mode 100755 index 0000000..65bd9b6 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/empty_trash.go @@ -0,0 +1,9 @@ +package src + +// EmptyTrash will permanently delete all trashed files/folders from Yandex Disk +func (c *Client) EmptyTrash() error { + fullURL := RootAddr + fullURL += "/v1/disk/trash/resources" + + return c.PerformDelete(fullURL) +} diff --git a/.rclone_repo/backend/yandex/api/error.go b/.rclone_repo/backend/yandex/api/error.go new file mode 100755 index 0000000..70e1475 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/error.go @@ -0,0 +1,84 @@ +package src + +//from yadisk + +import ( + "encoding/json" + "fmt" + "io" + "net/http" +) + +// ErrorResponse represents erroneous API response. +// Implements go's built in `error`. +type ErrorResponse struct { + ErrorName string `json:"error"` + Description string `json:"description"` + Message string `json:"message"` + + StatusCode int `json:""` +} + +func (e *ErrorResponse) Error() string { + return fmt.Sprintf("[%d - %s] %s (%s)", e.StatusCode, e.ErrorName, e.Description, e.Message) +} + +// ProccessErrorResponse tries to represent data passed as +// an ErrorResponse object. +func ProccessErrorResponse(data io.Reader) (*ErrorResponse, error) { + dec := json.NewDecoder(data) + var errorResponse ErrorResponse + + if err := dec.Decode(&errorResponse); err == io.EOF { + // ok + } else if err != nil { + return nil, err + } + + // TODO: check if there is any trash data after JSON and crash if there is. + + return &errorResponse, nil +} + +// CheckAPIError is a convenient function to turn erroneous +// API response into go error. It closes the Body on error. +func CheckAPIError(resp *http.Response) (err error) { + if resp.StatusCode >= 200 && resp.StatusCode < 400 { + return nil + } + + defer CheckClose(resp.Body, &err) + + errorResponse, err := ProccessErrorResponse(resp.Body) + if err != nil { + return err + } + errorResponse.StatusCode = resp.StatusCode + + return errorResponse +} + +// ProccessErrorString tries to represent data passed as +// an ErrorResponse object. +func ProccessErrorString(data string) (*ErrorResponse, error) { + var errorResponse ErrorResponse + if err := json.Unmarshal([]byte(data), &errorResponse); err == nil { + // ok + } else if err != nil { + return nil, err + } + + // TODO: check if there is any trash data after JSON and crash if there is. + + return &errorResponse, nil +} + +// ParseAPIError Parse json error response from API +func (c *Client) ParseAPIError(jsonErr string) (string, error) { //ErrorName + errorResponse, err := ProccessErrorString(jsonErr) + if err != nil { + return err.Error(), err + } + + return errorResponse.ErrorName, nil +} diff --git a/.rclone_repo/backend/yandex/api/errors.go b/.rclone_repo/backend/yandex/api/errors.go new file mode 100755 index 0000000..1f02788 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/errors.go @@ -0,0 +1,14 @@ +package src + +import "encoding/json" + +//DiskClientError struct +type DiskClientError struct { + Description string `json:"Description"` + Code string `json:"Error"` +} + +func (e DiskClientError) Error() string { + b, _ := json.Marshal(e) + return string(b) +} diff --git a/.rclone_repo/backend/yandex/api/files_resource_list.go b/.rclone_repo/backend/yandex/api/files_resource_list.go new file mode 100755 index 0000000..0af5429 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/files_resource_list.go @@ -0,0 +1,8 @@ +package src + +// FilesResourceListResponse struct is returned by the API for requests. +type FilesResourceListResponse struct { + Items []ResourceInfoResponse `json:"items"` + Limit *uint64 `json:"limit"` + Offset *uint64 `json:"offset"` +} diff --git a/.rclone_repo/backend/yandex/api/flat_file_list_request.go b/.rclone_repo/backend/yandex/api/flat_file_list_request.go new file mode 100755 index 0000000..ce047fa --- /dev/null +++ b/.rclone_repo/backend/yandex/api/flat_file_list_request.go @@ -0,0 +1,78 @@ +package src + +import ( + "encoding/json" + "strings" +) + +// FlatFileListRequest struct client for FlatFileList Request +type FlatFileListRequest struct { + client *Client + HTTPRequest *HTTPRequest +} + +// FlatFileListRequestOptions struct - options for request +type FlatFileListRequestOptions struct { + MediaType []MediaType + Limit *uint32 + Offset *uint32 + Fields []string + PreviewSize *PreviewSize + PreviewCrop *bool +} + +// Request get request +func (req *FlatFileListRequest) Request() *HTTPRequest { + return req.HTTPRequest +} + +// NewFlatFileListRequest create new FlatFileList Request +func (c *Client) NewFlatFileListRequest(options ...FlatFileListRequestOptions) *FlatFileListRequest { + var parameters = make(map[string]interface{}) + if len(options) > 0 { + opt := options[0] + if opt.Limit != nil { + parameters["limit"] = *opt.Limit + } + if opt.Offset != nil { + parameters["offset"] = *opt.Offset + } + if opt.Fields != nil { + parameters["fields"] = strings.Join(opt.Fields, ",") + } + if opt.PreviewSize != nil { + parameters["preview_size"] = opt.PreviewSize.String() + } + if opt.PreviewCrop != nil { + parameters["preview_crop"] = *opt.PreviewCrop + } + if opt.MediaType != nil { + var strMediaTypes = make([]string, len(opt.MediaType)) + for i, t := range opt.MediaType { + strMediaTypes[i] = t.String() + } + parameters["media_type"] = strings.Join(strMediaTypes, ",") + } + } + return &FlatFileListRequest{ + client: c, + HTTPRequest: createGetRequest(c, "/resources/files", parameters), + } +} + +// Exec run FlatFileList Request +func (req *FlatFileListRequest) Exec() (*FilesResourceListResponse, error) { + data, err := req.Request().run(req.client) + if err != nil { + return nil, err + } + var info FilesResourceListResponse + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + if cap(info.Items) == 0 { + info.Items = []ResourceInfoResponse{} + } + return &info, nil +} diff --git a/.rclone_repo/backend/yandex/api/http_request.go b/.rclone_repo/backend/yandex/api/http_request.go new file mode 100755 index 0000000..be04c84 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/http_request.go @@ -0,0 +1,24 @@ +package src + +// HTTPRequest struct +type HTTPRequest struct { + Method string + Path string + Parameters map[string]interface{} + Headers map[string][]string +} + +func createGetRequest(client *Client, path string, params map[string]interface{}) *HTTPRequest { + return createRequest(client, "GET", path, params) +} + +func createRequest(client *Client, method string, path string, parameters map[string]interface{}) *HTTPRequest { + var headers = make(map[string][]string) + headers["Authorization"] = []string{"OAuth " + client.token} + return &HTTPRequest{ + Method: method, + Path: path, + Parameters: parameters, + Headers: headers, + } +} diff --git a/.rclone_repo/backend/yandex/api/last_uploaded_resource_list.go b/.rclone_repo/backend/yandex/api/last_uploaded_resource_list.go new file mode 100755 index 0000000..8c667e6 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/last_uploaded_resource_list.go @@ -0,0 +1,7 @@ +package src + +// LastUploadedResourceListResponse struct +type LastUploadedResourceListResponse struct { + Items []ResourceInfoResponse `json:"items"` + Limit *uint64 `json:"limit"` +} diff --git a/.rclone_repo/backend/yandex/api/last_uploaded_resource_list_request.go b/.rclone_repo/backend/yandex/api/last_uploaded_resource_list_request.go new file mode 100755 index 0000000..f2c3fee --- /dev/null +++ b/.rclone_repo/backend/yandex/api/last_uploaded_resource_list_request.go @@ -0,0 +1,74 @@ +package src + +import ( + "encoding/json" + "strings" +) + +// LastUploadedResourceListRequest struct +type LastUploadedResourceListRequest struct { + client *Client + HTTPRequest *HTTPRequest +} + +// LastUploadedResourceListRequestOptions struct +type LastUploadedResourceListRequestOptions struct { + MediaType []MediaType + Limit *uint32 + Fields []string + PreviewSize *PreviewSize + PreviewCrop *bool +} + +// Request return request +func (req *LastUploadedResourceListRequest) Request() *HTTPRequest { + return req.HTTPRequest +} + +// NewLastUploadedResourceListRequest create new LastUploadedResourceList Request +func (c *Client) NewLastUploadedResourceListRequest(options ...LastUploadedResourceListRequestOptions) *LastUploadedResourceListRequest { + var parameters = make(map[string]interface{}) + if len(options) > 0 { + opt := options[0] + if opt.Limit != nil { + parameters["limit"] = opt.Limit + } + if opt.Fields != nil { + parameters["fields"] = strings.Join(opt.Fields, ",") + } + if opt.PreviewSize != nil { + parameters["preview_size"] = opt.PreviewSize.String() + } + if opt.PreviewCrop != nil { + parameters["preview_crop"] = opt.PreviewCrop + } + if opt.MediaType != nil { + var strMediaTypes = make([]string, len(opt.MediaType)) + for i, t := range opt.MediaType { + strMediaTypes[i] = t.String() + } + parameters["media_type"] = strings.Join(strMediaTypes, ",") + } + } + return &LastUploadedResourceListRequest{ + client: c, + HTTPRequest: createGetRequest(c, "/resources/last-uploaded", parameters), + } +} + +// Exec run LastUploadedResourceList Request +func (req *LastUploadedResourceListRequest) Exec() (*LastUploadedResourceListResponse, error) { + data, err := req.Request().run(req.client) + if err != nil { + return nil, err + } + var info LastUploadedResourceListResponse + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + if cap(info.Items) == 0 { + info.Items = []ResourceInfoResponse{} + } + return &info, nil +} diff --git a/.rclone_repo/backend/yandex/api/media_type.go b/.rclone_repo/backend/yandex/api/media_type.go new file mode 100755 index 0000000..5f913e0 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/media_type.go @@ -0,0 +1,144 @@ +package src + +// MediaType struct - media types +type MediaType struct { + mediaType string +} + +// Audio - media type +func (m *MediaType) Audio() *MediaType { + return &MediaType{ + mediaType: "audio", + } +} + +// Backup - media type +func (m *MediaType) Backup() *MediaType { + return &MediaType{ + mediaType: "backup", + } +} + +// Book - media type +func (m *MediaType) Book() *MediaType { + return &MediaType{ + mediaType: "book", + } +} + +// Compressed - media type +func (m *MediaType) Compressed() *MediaType { + return &MediaType{ + mediaType: "compressed", + } +} + +// Data - media type +func (m *MediaType) Data() *MediaType { + return &MediaType{ + mediaType: "data", + } +} + +// Development - media type +func (m *MediaType) Development() *MediaType { + return &MediaType{ + mediaType: "development", + } +} + +// Diskimage - media type +func (m *MediaType) Diskimage() *MediaType { + return &MediaType{ + mediaType: "diskimage", + } +} + +// Document - media type +func (m *MediaType) Document() *MediaType { + return &MediaType{ + mediaType: "document", + } +} + +// Encoded - media type +func (m *MediaType) Encoded() *MediaType { + return &MediaType{ + mediaType: "encoded", + } +} + +// Executable - media type +func (m *MediaType) Executable() *MediaType { + return &MediaType{ + mediaType: "executable", + } +} + +// Flash - media type +func (m *MediaType) Flash() *MediaType { + return &MediaType{ + mediaType: "flash", + } +} + +// Font - media type +func (m *MediaType) Font() *MediaType { + return &MediaType{ + mediaType: "font", + } +} + +// Image - media type +func (m *MediaType) Image() *MediaType { + return &MediaType{ + mediaType: "image", + } +} + +// Settings - media type +func (m *MediaType) Settings() *MediaType { + return &MediaType{ + mediaType: "settings", + } +} + +// Spreadsheet - media type +func (m *MediaType) Spreadsheet() *MediaType { + return &MediaType{ + mediaType: "spreadsheet", + } +} + +// Text - media type +func (m *MediaType) Text() *MediaType { + return &MediaType{ + mediaType: "text", + } +} + +// Unknown - media type +func (m *MediaType) Unknown() *MediaType { + return &MediaType{ + mediaType: "unknown", + } +} + +// Video - media type +func (m *MediaType) Video() *MediaType { + return &MediaType{ + mediaType: "video", + } +} + +// Web - media type +func (m *MediaType) Web() *MediaType { + return &MediaType{ + mediaType: "web", + } +} + +// String - media type +func (m *MediaType) String() string { + return m.mediaType +} diff --git a/.rclone_repo/backend/yandex/api/mkdir.go b/.rclone_repo/backend/yandex/api/mkdir.go new file mode 100755 index 0000000..e9703f4 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/mkdir.go @@ -0,0 +1,21 @@ +package src + +import ( + "net/url" +) + +// Mkdir will make specified folder on Yandex Disk +func (c *Client) Mkdir(remotePath string) (int, string, error) { + + values := url.Values{} + values.Add("path", remotePath) // only one current folder will be created. Not all the folders in the path. + urlPath := "/v1/disk/resources?" + values.Encode() + fullURL := RootAddr + if urlPath[:1] != "/" { + fullURL += "/" + urlPath + } else { + fullURL += urlPath + } + + return c.PerformMkdir(fullURL) +} diff --git a/.rclone_repo/backend/yandex/api/performdelete.go b/.rclone_repo/backend/yandex/api/performdelete.go new file mode 100755 index 0000000..bcdbcaf --- /dev/null +++ b/.rclone_repo/backend/yandex/api/performdelete.go @@ -0,0 +1,35 @@ +package src + +import ( + "io/ioutil" + "net/http" + + "github.com/pkg/errors" +) + +// PerformDelete does the actual delete via DELETE request. +func (c *Client) PerformDelete(url string) error { + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return err + } + + //set access token and headers + c.setRequestScope(req) + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + + //204 - resource deleted. + //202 - folder not empty, content will be deleted soon (async delete). + if resp.StatusCode != 204 && resp.StatusCode != 202 { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + return errors.Errorf("delete error [%d]: %s", resp.StatusCode, string(body)) + } + return nil +} diff --git a/.rclone_repo/backend/yandex/api/performdownload.go b/.rclone_repo/backend/yandex/api/performdownload.go new file mode 100755 index 0000000..93deef4 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/performdownload.go @@ -0,0 +1,40 @@ +package src + +import ( + "io" + "io/ioutil" + "net/http" + + "github.com/pkg/errors" +) + +// PerformDownload does the actual download via unscoped GET request. +func (c *Client) PerformDownload(url string, headers map[string]string) (out io.ReadCloser, err error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + // Set any extra headers + for k, v := range headers { + req.Header.Set(k, v) + } + + //c.setRequestScope(req) + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + + _, isRanging := req.Header["Range"] + if !(resp.StatusCode == http.StatusOK || (isRanging && resp.StatusCode == http.StatusPartialContent)) { + defer CheckClose(resp.Body, &err) + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return nil, errors.Errorf("download error [%d]: %s", resp.StatusCode, string(body)) + } + return resp.Body, err +} diff --git a/.rclone_repo/backend/yandex/api/performmkdir.go b/.rclone_repo/backend/yandex/api/performmkdir.go new file mode 100755 index 0000000..811beb2 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/performmkdir.go @@ -0,0 +1,34 @@ +package src + +import ( + "io/ioutil" + "net/http" + + "github.com/pkg/errors" +) + +// PerformMkdir does the actual mkdir via PUT request. +func (c *Client) PerformMkdir(url string) (int, string, error) { + req, err := http.NewRequest("PUT", url, nil) + if err != nil { + return 0, "", err + } + + //set access token and headers + c.setRequestScope(req) + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return 0, "", err + } + + if resp.StatusCode != 201 { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return 0, "", err + } + //third parameter is the json error response body + return resp.StatusCode, string(body), errors.Errorf("create folder error [%d]: %s", resp.StatusCode, string(body)) + } + return resp.StatusCode, "", nil +} diff --git a/.rclone_repo/backend/yandex/api/performupload.go b/.rclone_repo/backend/yandex/api/performupload.go new file mode 100755 index 0000000..3faff08 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/performupload.go @@ -0,0 +1,38 @@ +package src + +//from yadisk + +import ( + "io" + "io/ioutil" + "net/http" + + "github.com/pkg/errors" +) + +// PerformUpload does the actual upload via unscoped PUT request. +func (c *Client) PerformUpload(url string, data io.Reader, contentType string) (err error) { + req, err := http.NewRequest("PUT", url, data) + if err != nil { + return err + } + req.Header.Set("Content-Type", contentType) + + //c.setRequestScope(req) + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer CheckClose(resp.Body, &err) + + if resp.StatusCode != 201 { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return errors.Errorf("upload error [%d]: %s", resp.StatusCode, string(body)) + } + return nil +} diff --git a/.rclone_repo/backend/yandex/api/preview_size.go b/.rclone_repo/backend/yandex/api/preview_size.go new file mode 100755 index 0000000..3fba5c8 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/preview_size.go @@ -0,0 +1,75 @@ +package src + +import "fmt" + +// PreviewSize struct +type PreviewSize struct { + size string +} + +// PredefinedSizeS - set preview size +func (s *PreviewSize) PredefinedSizeS() *PreviewSize { + return &PreviewSize{ + size: "S", + } +} + +// PredefinedSizeM - set preview size +func (s *PreviewSize) PredefinedSizeM() *PreviewSize { + return &PreviewSize{ + size: "M", + } +} + +// PredefinedSizeL - set preview size +func (s *PreviewSize) PredefinedSizeL() *PreviewSize { + return &PreviewSize{ + size: "L", + } +} + +// PredefinedSizeXL - set preview size +func (s *PreviewSize) PredefinedSizeXL() *PreviewSize { + return &PreviewSize{ + size: "XL", + } +} + +// PredefinedSizeXXL - set preview size +func (s *PreviewSize) PredefinedSizeXXL() *PreviewSize { + return &PreviewSize{ + size: "XXL", + } +} + +// PredefinedSizeXXXL - set preview size +func (s *PreviewSize) PredefinedSizeXXXL() *PreviewSize { + return &PreviewSize{ + size: "XXXL", + } +} + +// ExactWidth - set preview size +func (s *PreviewSize) ExactWidth(width uint32) *PreviewSize { + return &PreviewSize{ + size: fmt.Sprintf("%dx", width), + } +} + +// ExactHeight - set preview size +func (s *PreviewSize) ExactHeight(height uint32) *PreviewSize { + return &PreviewSize{ + size: fmt.Sprintf("x%d", height), + } +} + +// ExactSize - set preview size +func (s *PreviewSize) ExactSize(width uint32, height uint32) *PreviewSize { + return &PreviewSize{ + size: fmt.Sprintf("%dx%d", width, height), + } +} + +func (s *PreviewSize) String() string { + return s.size +} diff --git a/.rclone_repo/backend/yandex/api/resource.go b/.rclone_repo/backend/yandex/api/resource.go new file mode 100755 index 0000000..6d9adb5 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/resource.go @@ -0,0 +1,19 @@ +package src + +//ResourceInfoResponse struct is returned by the API for metedata requests. +type ResourceInfoResponse struct { + PublicKey string `json:"public_key"` + Name string `json:"name"` + Created string `json:"created"` + CustomProperties map[string]interface{} `json:"custom_properties"` + Preview string `json:"preview"` + PublicURL string `json:"public_url"` + OriginPath string `json:"origin_path"` + Modified string `json:"modified"` + Path string `json:"path"` + Md5 string `json:"md5"` + ResourceType string `json:"type"` + MimeType string `json:"mime_type"` + Size uint64 `json:"size"` + Embedded *ResourceListResponse `json:"_embedded"` +} diff --git a/.rclone_repo/backend/yandex/api/resource_info_request.go b/.rclone_repo/backend/yandex/api/resource_info_request.go new file mode 100755 index 0000000..5ad6017 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/resource_info_request.go @@ -0,0 +1,45 @@ +package src + +import "encoding/json" + +// ResourceInfoRequest struct +type ResourceInfoRequest struct { + client *Client + HTTPRequest *HTTPRequest +} + +// Request of ResourceInfoRequest +func (req *ResourceInfoRequest) Request() *HTTPRequest { + return req.HTTPRequest +} + +// NewResourceInfoRequest create new ResourceInfo Request +func (c *Client) NewResourceInfoRequest(path string, options ...ResourceInfoRequestOptions) *ResourceInfoRequest { + return &ResourceInfoRequest{ + client: c, + HTTPRequest: createResourceInfoRequest(c, "/resources", path, options...), + } +} + +// Exec run ResourceInfo Request +func (req *ResourceInfoRequest) Exec() (*ResourceInfoResponse, error) { + data, err := req.Request().run(req.client) + if err != nil { + return nil, err + } + + var info ResourceInfoResponse + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + if info.CustomProperties == nil { + info.CustomProperties = make(map[string]interface{}) + } + if info.Embedded != nil { + if cap(info.Embedded.Items) == 0 { + info.Embedded.Items = []ResourceInfoResponse{} + } + } + return &info, nil +} diff --git a/.rclone_repo/backend/yandex/api/resource_info_request_helpers.go b/.rclone_repo/backend/yandex/api/resource_info_request_helpers.go new file mode 100755 index 0000000..8817187 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/resource_info_request_helpers.go @@ -0,0 +1,33 @@ +package src + +import "strings" + +func createResourceInfoRequest(c *Client, + apiPath string, + path string, + options ...ResourceInfoRequestOptions) *HTTPRequest { + var parameters = make(map[string]interface{}) + parameters["path"] = path + if len(options) > 0 { + opt := options[0] + if opt.SortMode != nil { + parameters["sort"] = opt.SortMode.String() + } + if opt.Limit != nil { + parameters["limit"] = *opt.Limit + } + if opt.Offset != nil { + parameters["offset"] = *opt.Offset + } + if opt.Fields != nil { + parameters["fields"] = strings.Join(opt.Fields, ",") + } + if opt.PreviewSize != nil { + parameters["preview_size"] = opt.PreviewSize.String() + } + if opt.PreviewCrop != nil { + parameters["preview_crop"] = *opt.PreviewCrop + } + } + return createGetRequest(c, apiPath, parameters) +} diff --git a/.rclone_repo/backend/yandex/api/resource_info_request_options.go b/.rclone_repo/backend/yandex/api/resource_info_request_options.go new file mode 100755 index 0000000..ffe07d4 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/resource_info_request_options.go @@ -0,0 +1,11 @@ +package src + +// ResourceInfoRequestOptions struct +type ResourceInfoRequestOptions struct { + SortMode *SortMode + Limit *uint32 + Offset *uint32 + Fields []string + PreviewSize *PreviewSize + PreviewCrop *bool +} diff --git a/.rclone_repo/backend/yandex/api/resource_list.go b/.rclone_repo/backend/yandex/api/resource_list.go new file mode 100755 index 0000000..f15caf1 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/resource_list.go @@ -0,0 +1,12 @@ +package src + +// ResourceListResponse struct +type ResourceListResponse struct { + Sort *SortMode `json:"sort"` + PublicKey string `json:"public_key"` + Items []ResourceInfoResponse `json:"items"` + Path string `json:"path"` + Limit *uint64 `json:"limit"` + Offset *uint64 `json:"offset"` + Total *uint64 `json:"total"` +} diff --git a/.rclone_repo/backend/yandex/api/sort_mode.go b/.rclone_repo/backend/yandex/api/sort_mode.go new file mode 100755 index 0000000..41e74e6 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/sort_mode.go @@ -0,0 +1,79 @@ +package src + +import "strings" + +// SortMode struct - sort mode +type SortMode struct { + mode string +} + +// Default - sort mode +func (m *SortMode) Default() *SortMode { + return &SortMode{ + mode: "", + } +} + +// ByName - sort mode +func (m *SortMode) ByName() *SortMode { + return &SortMode{ + mode: "name", + } +} + +// ByPath - sort mode +func (m *SortMode) ByPath() *SortMode { + return &SortMode{ + mode: "path", + } +} + +// ByCreated - sort mode +func (m *SortMode) ByCreated() *SortMode { + return &SortMode{ + mode: "created", + } +} + +// ByModified - sort mode +func (m *SortMode) ByModified() *SortMode { + return &SortMode{ + mode: "modified", + } +} + +// BySize - sort mode +func (m *SortMode) BySize() *SortMode { + return &SortMode{ + mode: "size", + } +} + +// Reverse - sort mode +func (m *SortMode) Reverse() *SortMode { + if strings.HasPrefix(m.mode, "-") { + return &SortMode{ + mode: m.mode[1:], + } + } + return &SortMode{ + mode: "-" + m.mode, + } +} + +func (m *SortMode) String() string { + return m.mode +} + +// UnmarshalJSON sort mode +func (m *SortMode) UnmarshalJSON(value []byte) error { + if value == nil || len(value) == 0 { + m.mode = "" + return nil + } + m.mode = string(value) + if strings.HasPrefix(m.mode, "\"") && strings.HasSuffix(m.mode, "\"") { + m.mode = m.mode[1 : len(m.mode)-1] + } + return nil +} diff --git a/.rclone_repo/backend/yandex/api/trash_resource_info_request.go b/.rclone_repo/backend/yandex/api/trash_resource_info_request.go new file mode 100755 index 0000000..3911223 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/trash_resource_info_request.go @@ -0,0 +1,45 @@ +package src + +import "encoding/json" + +// TrashResourceInfoRequest struct +type TrashResourceInfoRequest struct { + client *Client + HTTPRequest *HTTPRequest +} + +// Request of TrashResourceInfoRequest struct +func (req *TrashResourceInfoRequest) Request() *HTTPRequest { + return req.HTTPRequest +} + +// NewTrashResourceInfoRequest create new TrashResourceInfo Request +func (c *Client) NewTrashResourceInfoRequest(path string, options ...ResourceInfoRequestOptions) *TrashResourceInfoRequest { + return &TrashResourceInfoRequest{ + client: c, + HTTPRequest: createResourceInfoRequest(c, "/trash/resources", path, options...), + } +} + +// Exec run TrashResourceInfo Request +func (req *TrashResourceInfoRequest) Exec() (*ResourceInfoResponse, error) { + data, err := req.Request().run(req.client) + if err != nil { + return nil, err + } + + var info ResourceInfoResponse + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + if info.CustomProperties == nil { + info.CustomProperties = make(map[string]interface{}) + } + if info.Embedded != nil { + if cap(info.Embedded.Items) == 0 { + info.Embedded.Items = []ResourceInfoResponse{} + } + } + return &info, nil +} diff --git a/.rclone_repo/backend/yandex/api/upload.go b/.rclone_repo/backend/yandex/api/upload.go new file mode 100755 index 0000000..f8d3f19 --- /dev/null +++ b/.rclone_repo/backend/yandex/api/upload.go @@ -0,0 +1,71 @@ +package src + +//from yadisk + +import ( + "encoding/json" + "io" + "net/url" + "strconv" +) + +// UploadResponse struct is returned by the API for upload request. +type UploadResponse struct { + HRef string `json:"href"` + Method string `json:"method"` + Templated bool `json:"templated"` +} + +// Upload will put specified data to Yandex.Disk. +func (c *Client) Upload(data io.Reader, remotePath string, overwrite bool, contentType string) error { + ur, err := c.UploadRequest(remotePath, overwrite) + if err != nil { + return err + } + + return c.PerformUpload(ur.HRef, data, contentType) +} + +// UploadRequest will make an upload request and return a URL to upload data to. +func (c *Client) UploadRequest(remotePath string, overwrite bool) (ur *UploadResponse, err error) { + values := url.Values{} + values.Add("path", remotePath) + values.Add("overwrite", strconv.FormatBool(overwrite)) + + req, err := c.scopedRequest("GET", "/v1/disk/resources/upload?"+values.Encode(), nil) + if err != nil { + return nil, err + } + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + if err := CheckAPIError(resp); err != nil { + return nil, err + } + defer CheckClose(resp.Body, &err) + + ur, err = ParseUploadResponse(resp.Body) + if err != nil { + return nil, err + } + + return ur, nil +} + +// ParseUploadResponse tries to read and parse UploadResponse struct. +func ParseUploadResponse(data io.Reader) (*UploadResponse, error) { + dec := json.NewDecoder(data) + var ur UploadResponse + + if err := dec.Decode(&ur); err == io.EOF { + // ok + } else if err != nil { + return nil, err + } + + // TODO: check if there is any trash data after JSON and crash if there is. + + return &ur, nil +} diff --git a/.rclone_repo/backend/yandex/yandex.go b/.rclone_repo/backend/yandex/yandex.go new file mode 100755 index 0000000..0979c4e --- /dev/null +++ b/.rclone_repo/backend/yandex/yandex.go @@ -0,0 +1,692 @@ +// Package yandex provides an interface to the Yandex Disk storage. +// +// dibu28 github.com/dibu28 +package yandex + +import ( + "encoding/json" + "fmt" + "io" + "log" + "path" + "path/filepath" + "strings" + "time" + + yandex "github.com/ncw/rclone/backend/yandex/api" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/oauthutil" + "github.com/ncw/rclone/lib/readers" + "github.com/pkg/errors" + "golang.org/x/oauth2" +) + +//oAuth +const ( + rcloneClientID = "ac39b43b9eba4cae8ffb788c06d816a8" + rcloneEncryptedClientSecret = "EfyyNZ3YUEwXM5yAhi72G9YwKn2mkFrYwJNS7cY0TJAhFlX9K-uJFbGlpO-RYjrJ" +) + +// Globals +var ( + // Description of how to auth for this app + oauthConfig = &oauth2.Config{ + Endpoint: oauth2.Endpoint{ + AuthURL: "https://oauth.yandex.com/authorize", //same as https://oauth.yandex.ru/authorize + TokenURL: "https://oauth.yandex.com/token", //same as https://oauth.yandex.ru/token + }, + ClientID: rcloneClientID, + ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), + RedirectURL: oauthutil.RedirectURL, + } +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "yandex", + Description: "Yandex Disk", + NewFs: NewFs, + Config: func(name string, m configmap.Mapper) { + err := oauthutil.Config("yandex", name, m, oauthConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + }, + Options: []fs.Option{{ + Name: config.ConfigClientID, + Help: "Yandex Client Id\nLeave blank normally.", + }, { + Name: config.ConfigClientSecret, + Help: "Yandex Client Secret\nLeave blank normally.", + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + Token string `config:"token"` +} + +// Fs represents a remote yandex +type Fs struct { + name string + root string // root path + opt Options // parsed options + features *fs.Features // optional features + yd *yandex.Client // client for rest api + diskRoot string // root path with "disk:/" container name +} + +// Object describes a swift object +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + md5sum string // The MD5Sum of the object + bytes uint64 // Bytes in the object + modTime time.Time // Modified time of the object + mimeType string // Content type according to the server +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("Yandex %s", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// read access token from ConfigFile string +func getAccessToken(opt *Options) (*oauth2.Token, error) { + //Get access token from config string + decoder := json.NewDecoder(strings.NewReader(opt.Token)) + var result *oauth2.Token + err := decoder.Decode(&result) + if err != nil { + return nil, err + } + return result, nil +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + //read access token from config + token, err := getAccessToken(opt) + if err != nil { + return nil, err + } + + //create new client + yandexDisk := yandex.NewClient(token.AccessToken, fshttp.NewClient(fs.Config)) + + f := &Fs{ + name: name, + opt: *opt, + yd: yandexDisk, + } + f.features = (&fs.Features{ + ReadMimeType: true, + WriteMimeType: true, + CanHaveEmptyDirectories: true, + }).Fill(f) + f.setRoot(root) + + // Check to see if the object exists and is a file + //request object meta info + var opt2 yandex.ResourceInfoRequestOptions + if ResourceInfoResponse, err := yandexDisk.NewResourceInfoRequest(root, opt2).Exec(); err != nil { + //return err + } else { + if ResourceInfoResponse.ResourceType == "file" { + rootDir := path.Dir(root) + if rootDir == "." { + rootDir = "" + } + f.setRoot(rootDir) + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + } + + return f, nil +} + +// Sets root in f +func (f *Fs) setRoot(root string) { + //Set root path + f.root = strings.Trim(root, "/") + //Set disk root path. + //Adding "disk:" to root path as all paths on disk start with it + var diskRoot string + if f.root == "" { + diskRoot = "disk:/" + } else { + diskRoot = "disk:/" + f.root + "/" + } + f.diskRoot = diskRoot +} + +// Convert a list item into a DirEntry +func (f *Fs) itemToDirEntry(remote string, object *yandex.ResourceInfoResponse) (fs.DirEntry, error) { + switch object.ResourceType { + case "dir": + t, err := time.Parse(time.RFC3339Nano, object.Modified) + if err != nil { + return nil, errors.Wrap(err, "error parsing time in directory item") + } + d := fs.NewDir(remote, t).SetSize(int64(object.Size)) + return d, nil + case "file": + o, err := f.newObjectWithInfo(remote, object) + if err != nil { + return nil, err + } + return o, nil + default: + fs.Debugf(f, "Unknown resource type %q", object.ResourceType) + } + return nil, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + //request object meta info + var opt yandex.ResourceInfoRequestOptions + root := f.diskRoot + if dir != "" { + root += dir + "/" + } + var limit uint32 = 1000 // max number of object per request + var itemsCount uint32 //number of items per page in response + var offset uint32 //for the next page of request + opt.Limit = &limit + opt.Offset = &offset + + //query each page of list until itemCount is less then limit + for { + ResourceInfoResponse, err := f.yd.NewResourceInfoRequest(root, opt).Exec() + if err != nil { + yErr, ok := err.(yandex.DiskClientError) + if ok && yErr.Code == "DiskNotFoundError" { + return nil, fs.ErrorDirNotFound + } + return nil, err + } + itemsCount = uint32(len(ResourceInfoResponse.Embedded.Items)) + + if ResourceInfoResponse.ResourceType == "dir" { + //list all subdirs + for _, element := range ResourceInfoResponse.Embedded.Items { + remote := path.Join(dir, element.Name) + entry, err := f.itemToDirEntry(remote, &element) + if err != nil { + return nil, err + } + if entry != nil { + entries = append(entries, entry) + } + } + } + + //offset for the next page of items + offset += itemsCount + //check if we reached end of list + if itemsCount < limit { + break + } + } + return entries, nil +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + //request files list. list is divided into pages. We send request for each page + //items per page is limited by limit + //TODO may be add config parameter for the items per page limit + var limit uint32 = 1000 // max number of object per request + var itemsCount uint32 //number of items per page in response + var offset uint32 //for the next page of request + // yandex disk api request options + var opt yandex.FlatFileListRequestOptions + opt.Limit = &limit + opt.Offset = &offset + prefix := f.diskRoot + if dir != "" { + prefix += dir + "/" + } + //query each page of list until itemCount is less then limit + for { + //send request + info, err := f.yd.NewFlatFileListRequest(opt).Exec() + if err != nil { + yErr, ok := err.(yandex.DiskClientError) + if ok && yErr.Code == "DiskNotFoundError" { + return fs.ErrorDirNotFound + } + return err + } + itemsCount = uint32(len(info.Items)) + + //list files + entries := make(fs.DirEntries, 0, len(info.Items)) + for _, item := range info.Items { + // filter file list and get only files we need + if strings.HasPrefix(item.Path, prefix) { + //trim root folder from filename + var name = strings.TrimPrefix(item.Path, f.diskRoot) + entry, err := f.itemToDirEntry(name, &item) + if err != nil { + return err + } + if entry != nil { + entries = append(entries, entry) + } + } + } + // send the listing + err = callback(entries) + if err != nil { + return err + } + + //offset for the next page of items + offset += itemsCount + //check if we reached end of list + if itemsCount < limit { + break + } + } + return nil +} + +// NewObject finds the Object at remote. If it can't be found it +// returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + return f.newObjectWithInfo(remote, nil) +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(remote string, info *yandex.ResourceInfoResponse) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + err = o.setMetaData(info) + } else { + err = o.readMetaData() + } + if err != nil { + return nil, err + } + return o, nil +} + +// setMetaData sets the fs data from a storage.Object +func (o *Object) setMetaData(info *yandex.ResourceInfoResponse) (err error) { + if info.ResourceType != "file" { + return errors.Wrapf(fs.ErrorNotAFile, "%q", o.remote) + } + o.bytes = info.Size + o.md5sum = info.Md5 + o.mimeType = info.MimeType + + var modTimeString string + modTimeObj, ok := info.CustomProperties["rclone_modified"] + if ok { + // read modTime from rclone_modified custom_property of object + modTimeString, ok = modTimeObj.(string) + } + if !ok { + // read modTime from Modified property of object as a fallback + modTimeString = info.Modified + } + t, err := time.Parse(time.RFC3339Nano, modTimeString) + if err != nil { + return errors.Wrapf(err, "failed to parse modtime from %q", modTimeString) + } + o.modTime = t + return nil +} + +// readMetaData gets the info if it hasn't already been fetched +func (o *Object) readMetaData() (err error) { + // exit if already fetched + if !o.modTime.IsZero() { + return nil + } + + //request meta info + var opt2 yandex.ResourceInfoRequestOptions + ResourceInfoResponse, err := o.fs.yd.NewResourceInfoRequest(o.remotePath(), opt2).Exec() + if err != nil { + if dcErr, ok := err.(yandex.DiskClientError); ok { + if dcErr.Code == "DiskNotFoundError" { + return fs.ErrorObjectNotFound + } + } + return err + } + return o.setMetaData(ResourceInfoResponse) +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime() + + o := &Object{ + fs: f, + remote: remote, + bytes: uint64(size), + modTime: modTime, + } + //TODO maybe read metadata after upload to check if file uploaded successfully + return o, o.Update(in, src, options...) +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(in, src, options...) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(dir string) error { + root := f.diskRoot + if dir != "" { + root += dir + "/" + } + return mkDirFullPath(f.yd, root) +} + +// Rmdir deletes the container +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(dir string) error { + return f.purgeCheck(dir, true) +} + +// purgeCheck remotes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(dir string, check bool) error { + root := f.diskRoot + if dir != "" { + root += dir + "/" + } + if check { + //to comply with rclone logic we check if the directory is empty before delete. + //send request to get list of objects in this directory. + var opt yandex.ResourceInfoRequestOptions + ResourceInfoResponse, err := f.yd.NewResourceInfoRequest(root, opt).Exec() + if err != nil { + return errors.Wrap(err, "rmdir failed") + } + if len(ResourceInfoResponse.Embedded.Items) != 0 { + return errors.New("rmdir failed: directory not empty") + } + } + //delete directory + return f.yd.Delete(root, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return time.Nanosecond +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + return f.purgeCheck("", false) +} + +// CleanUp permanently deletes all trashed files/folders +func (f *Fs) CleanUp() error { + return f.yd.EmptyTrash() +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the Md5sum of an object returning a lowercase hex string +func (o *Object) Hash(t hash.Type) (string, error) { + if t != hash.MD5 { + return "", hash.ErrUnsupported + } + return o.md5sum, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + var size = int64(o.bytes) //need to cast from uint64 in yandex disk to int64 in rclone. can cause overflow + return size +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// Open an object for read +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + return o.fs.yd.Download(o.remotePath(), fs.OpenOptionHeaders(options)) +} + +// Remove an object +func (o *Object) Remove() error { + return o.fs.yd.Delete(o.remotePath(), true) +} + +// SetModTime sets the modification time of the local fs object +// +// Commits the datastore +func (o *Object) SetModTime(modTime time.Time) error { + remote := o.remotePath() + // set custom_property 'rclone_modified' of object to modTime + err := o.fs.yd.SetCustomProperty(remote, "rclone_modified", modTime.Format(time.RFC3339Nano)) + if err != nil { + return err + } + o.modTime = modTime + return nil +} + +// Storable returns whether this object is storable +func (o *Object) Storable() bool { + return true +} + +// Returns the remote path for the object +func (o *Object) remotePath() string { + return o.fs.diskRoot + o.remote +} + +// Update the already existing object +// +// Copy the reader into the object updating modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in0 io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + in := readers.NewCountingReader(in0) + modTime := src.ModTime() + + remote := o.remotePath() + //create full path to file before upload. + err1 := mkDirFullPath(o.fs.yd, remote) + if err1 != nil { + return err1 + } + //upload file + overwrite := true //overwrite existing file + mimeType := fs.MimeType(src) + err := o.fs.yd.Upload(in, remote, overwrite, mimeType) + if err == nil { + //if file uploaded sucessfully then return metadata + o.bytes = in.BytesRead() + o.modTime = modTime + o.md5sum = "" // according to unit tests after put the md5 is empty. + //and set modTime of uploaded file + err = o.SetModTime(modTime) + } + return err +} + +// utility funcs------------------------------------------------------------------- + +// mkDirExecute execute mkdir +func mkDirExecute(client *yandex.Client, path string) (int, string, error) { + statusCode, jsonErrorString, err := client.Mkdir(path) + if statusCode == 409 { // dir already exist + return statusCode, jsonErrorString, err + } + if statusCode == 201 { // dir was created + return statusCode, jsonErrorString, err + } + if err != nil { + // error creating directory + return statusCode, jsonErrorString, errors.Wrap(err, "failed to create folder") + } + return 0, "", nil +} + +//mkDirFullPath Creates Each Directory in the path if needed. Send request once for every directory in the path. +func mkDirFullPath(client *yandex.Client, path string) error { + //trim filename from path + dirString := strings.TrimSuffix(path, filepath.Base(path)) + //trim "disk:/" from path + dirString = strings.TrimPrefix(dirString, "disk:/") + + //1 Try to create directory first + if _, jsonErrorString, err := mkDirExecute(client, dirString); err != nil { + er2, _ := client.ParseAPIError(jsonErrorString) + if er2 != "DiskPathPointsToExistentDirectoryError" { + //2 if it fails then create all directories in the path from root. + dirs := strings.Split(dirString, "/") //path separator / + var mkdirpath = "/" //path separator / + for _, element := range dirs { + if element != "" { + mkdirpath += element + "/" //path separator / + _, _, err2 := mkDirExecute(client, mkdirpath) + if err2 != nil { + //we continue even if some directories exist. + } + } + } + } + } + return nil +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType() string { + err := o.readMetaData() + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return "" + } + return o.mimeType +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.CleanUpper = (*Fs)(nil) + _ fs.PutStreamer = (*Fs)(nil) + _ fs.ListRer = (*Fs)(nil) + //_ fs.Copier = (*Fs)(nil) + _ fs.ListRer = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = &Object{} +) diff --git a/.rclone_repo/backend/yandex/yandex_test.go b/.rclone_repo/backend/yandex/yandex_test.go new file mode 100755 index 0000000..4dcf976 --- /dev/null +++ b/.rclone_repo/backend/yandex/yandex_test.go @@ -0,0 +1,17 @@ +// Test Yandex filesystem interface +package yandex_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/yandex" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestYandex:", + NilObject: (*yandex.Object)(nil), + }) +} diff --git a/.rclone_repo/bin/cross-compile.go b/.rclone_repo/bin/cross-compile.go new file mode 100755 index 0000000..8e1a1e0 --- /dev/null +++ b/.rclone_repo/bin/cross-compile.go @@ -0,0 +1,297 @@ +// +build ignore + +// Cross compile rclone - in go because I hate bash ;-) + +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "sync" + "text/template" + "time" +) + +var ( + // Flags + debug = flag.Bool("d", false, "Print commands instead of running them.") + parallel = flag.Int("parallel", runtime.NumCPU(), "Number of commands to run in parallel.") + copyAs = flag.String("release", "", "Make copies of the releases with this name") + gitLog = flag.String("git-log", "", "git log to include as well") + include = flag.String("include", "^.*$", "os/arch regexp to include") + exclude = flag.String("exclude", "^$", "os/arch regexp to exclude") + cgo = flag.Bool("cgo", false, "Use cgo for the build") + noClean = flag.Bool("no-clean", false, "Don't clean the build directory before running.") + tags = flag.String("tags", "", "Space separated list of build tags") + compileOnly = flag.Bool("compile-only", false, "Just build the binary, not the zip.") +) + +// GOOS/GOARCH pairs we build for +var osarches = []string{ + "windows/386", + "windows/amd64", + "darwin/386", + "darwin/amd64", + "linux/386", + "linux/amd64", + "linux/arm", + "linux/arm64", + "linux/mips", + "linux/mipsle", + "freebsd/386", + "freebsd/amd64", + "freebsd/arm", + "netbsd/386", + "netbsd/amd64", + "netbsd/arm", + "openbsd/386", + "openbsd/amd64", + "plan9/386", + "plan9/amd64", + "solaris/amd64", +} + +// Special environment flags for a given arch +var archFlags = map[string][]string{ + "386": {"GO386=387"}, +} + +// runEnv - run a shell command with env +func runEnv(args, env []string) error { + if *debug { + args = append([]string{"echo"}, args...) + } + cmd := exec.Command(args[0], args[1:]...) + if env != nil { + cmd.Env = append(os.Environ(), env...) + } + if *debug { + log.Printf("args = %v, env = %v\n", args, cmd.Env) + } + out, err := cmd.CombinedOutput() + if err != nil { + log.Print("----------------------------") + log.Printf("Failed to run %v: %v", args, err) + log.Printf("Command output was:\n%s", out) + log.Print("----------------------------") + } + return err +} + +// run a shell command +func run(args ...string) { + err := runEnv(args, nil) + if err != nil { + log.Fatalf("Exiting after error: %v", err) + } +} + +// chdir or die +func chdir(dir string) { + err := os.Chdir(dir) + if err != nil { + log.Fatalf("Couldn't cd into %q: %v", dir, err) + } +} + +// substitute data from go template file in to file out +func substitute(inFile, outFile string, data interface{}) { + t, err := template.ParseFiles(inFile) + if err != nil { + log.Fatalf("Failed to read template file %q: %v %v", inFile, err) + } + out, err := os.Create(outFile) + if err != nil { + log.Fatalf("Failed to create output file %q: %v %v", outFile, err) + } + defer func() { + err := out.Close() + if err != nil { + log.Fatalf("Failed to close output file %q: %v %v", outFile, err) + } + }() + err = t.Execute(out, data) + if err != nil { + log.Fatalf("Failed to substitute template file %q: %v %v", inFile, err) + } +} + +// build the zip package return its name +func buildZip(dir string) string { + // Now build the zip + run("cp", "-a", "../MANUAL.txt", filepath.Join(dir, "README.txt")) + run("cp", "-a", "../MANUAL.html", filepath.Join(dir, "README.html")) + run("cp", "-a", "../rclone.1", dir) + if *gitLog != "" { + run("cp", "-a", *gitLog, dir) + } + zip := dir + ".zip" + run("zip", "-r9", zip, dir) + return zip +} + +// Build .deb and .rpm packages +// +// It returns a list of artifacts it has made +func buildDebAndRpm(dir, version, goarch string) []string { + // Make internal version number acceptable to .deb and .rpm + pkgVersion := version[1:] + pkgVersion = strings.Replace(pkgVersion, "β", "-beta", -1) + pkgVersion = strings.Replace(pkgVersion, "-", ".", -1) + + // Make nfpm.yaml from the template + substitute("../bin/nfpm.yaml", path.Join(dir, "nfpm.yaml"), map[string]string{ + "Version": pkgVersion, + "Arch": goarch, + }) + + // build them + var artifacts []string + for _, pkg := range []string{".deb", ".rpm"} { + artifact := dir + pkg + run("bash", "-c", "cd "+dir+" && nfpm -f nfpm.yaml pkg -t ../"+artifact) + artifacts = append(artifacts, artifact) + } + + return artifacts +} + +// build the binary in dir returning success or failure +func compileArch(version, goos, goarch, dir string) bool { + log.Printf("Compiling %s/%s", goos, goarch) + output := filepath.Join(dir, "rclone") + if goos == "windows" { + output += ".exe" + } + err := os.MkdirAll(dir, 0777) + if err != nil { + log.Fatalf("Failed to mkdir: %v", err) + } + args := []string{ + "go", "build", + "--ldflags", "-s -X github.com/ncw/rclone/fs.Version=" + version, + "-i", + "-o", output, + "-tags", *tags, + "..", + } + env := []string{ + "GOOS=" + goos, + "GOARCH=" + goarch, + } + if !*cgo { + env = append(env, "CGO_ENABLED=0") + } else { + env = append(env, "CGO_ENABLED=1") + } + if flags, ok := archFlags[goarch]; ok { + env = append(env, flags...) + } + err = runEnv(args, env) + if err != nil { + log.Printf("Error compiling %s/%s: %v", goos, goarch, err) + return false + } + if !*compileOnly { + artifacts := []string{buildZip(dir)} + // build a .deb and .rpm if appropriate + if goos == "linux" { + artifacts = append(artifacts, buildDebAndRpm(dir, version, goarch)...) + } + if *copyAs != "" { + for _, artifact := range artifacts { + run("ln", artifact, strings.Replace(artifact, "-"+version, "-"+*copyAs, 1)) + } + } + // tidy up + run("rm", "-rf", dir) + } + log.Printf("Done compiling %s/%s", goos, goarch) + return true +} + +func compile(version string) { + start := time.Now() + wg := new(sync.WaitGroup) + run := make(chan func(), *parallel) + for i := 0; i < *parallel; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for f := range run { + f() + } + }() + } + includeRe, err := regexp.Compile(*include) + if err != nil { + log.Fatalf("Bad -include regexp: %v", err) + } + excludeRe, err := regexp.Compile(*exclude) + if err != nil { + log.Fatalf("Bad -exclude regexp: %v", err) + } + compiled := 0 + var failuresMu sync.Mutex + var failures []string + for _, osarch := range osarches { + if excludeRe.MatchString(osarch) || !includeRe.MatchString(osarch) { + continue + } + parts := strings.Split(osarch, "/") + if len(parts) != 2 { + log.Fatalf("Bad osarch %q", osarch) + } + goos, goarch := parts[0], parts[1] + userGoos := goos + if goos == "darwin" { + userGoos = "osx" + } + dir := filepath.Join("rclone-" + version + "-" + userGoos + "-" + goarch) + run <- func() { + if !compileArch(version, goos, goarch, dir) { + failuresMu.Lock() + failures = append(failures, goos+"/"+goarch) + failuresMu.Unlock() + } + } + compiled++ + } + close(run) + wg.Wait() + log.Printf("Compiled %d arches in %v", compiled, time.Since(start)) + if len(failures) > 0 { + sort.Strings(failures) + log.Printf("%d compile failures:\n %s\n", len(failures), strings.Join(failures, "\n ")) + os.Exit(1) + } +} + +func main() { + flag.Parse() + args := flag.Args() + if len(args) != 1 { + log.Fatalf("Syntax: %s ", os.Args[0]) + } + version := args[0] + if !*noClean { + run("rm", "-rf", "build") + run("mkdir", "build") + } + chdir("build") + err := ioutil.WriteFile("version.txt", []byte(fmt.Sprintf("rclone %s\n", version)), 0666) + if err != nil { + log.Fatalf("Couldn't write version.txt: %v", err) + } + compile(version) +} diff --git a/.rclone_repo/bin/decrypt_names.py b/.rclone_repo/bin/decrypt_names.py new file mode 100755 index 0000000..1fec84c --- /dev/null +++ b/.rclone_repo/bin/decrypt_names.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +""" +This is a tool to decrypt file names in rclone logs. + +Pass two files in, the first should be a crypt mapping generated by + +rclone ls --crypt-show-mapping remote:path + +The second should be a log file that you want the paths decrypted in. + +Note that if the crypt mappings file is large it can take some time to +run. +""" + +import re +import sys + +# Crypt line +match_crypt = re.compile(r'NOTICE: (.*?): Encrypts to "(.*?)"$') + +def read_crypt_map(mapping_file): + """ + Read the crypt mapping file in, creating a dictionary of substitutions + """ + mapping = {} + with open(mapping_file) as fd: + for line in fd: + match = match_crypt.search(line) + if match: + plaintext, ciphertext = match.groups() + plaintexts = plaintext.split("/") + ciphertexts = ciphertext.split("/") + for plain, cipher in zip(plaintexts, ciphertexts): + mapping[cipher] = plain + return mapping + +def map_log_file(crypt_map, log_file): + """ + Substitute the crypt_map in the log file. + + This uses a straight forward O(N**2) algorithm. I tried using + regexps to speed it up but it made it slower! + """ + with open(log_file) as fd: + for line in fd: + for cipher, plain in crypt_map.iteritems(): + line = line.replace(cipher, plain) + sys.stdout.write(line) + +def main(): + if len(sys.argv) < 3: + print "Syntax: %s " % sys.argv[0] + raise SystemExit(1) + mapping_file, log_file = sys.argv[1:] + crypt_map = read_crypt_map(mapping_file) + map_log_file(crypt_map, log_file) + +if __name__ == "__main__": + main() diff --git a/.rclone_repo/bin/get-github-release.go b/.rclone_repo/bin/get-github-release.go new file mode 100755 index 0000000..61bd2f3 --- /dev/null +++ b/.rclone_repo/bin/get-github-release.go @@ -0,0 +1,264 @@ +// +build ignore + +// Get the latest release from a github project +// +// If GITHUB_USER and GITHUB_TOKEN are set then these will be used to +// authenticate the request which is useful to avoid rate limits. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +var ( + // Flags + install = flag.Bool("install", false, "Install the downloaded package using sudo dpkg -i.") + extract = flag.String("extract", "", "Extract the named executable from the .tar.gz and install into bindir.") + bindir = flag.String("bindir", defaultBinDir(), "Directory to install files downloaded with -extract.") + // Globals + matchProject = regexp.MustCompile(`^(\w+)/(\w+)$`) +) + +// A github release +// +// Made by pasting the JSON into https://mholt.github.io/json-to-go/ +type Release struct { + URL string `json:"url"` + AssetsURL string `json:"assets_url"` + UploadURL string `json:"upload_url"` + HTMLURL string `json:"html_url"` + ID int `json:"id"` + TagName string `json:"tag_name"` + TargetCommitish string `json:"target_commitish"` + Name string `json:"name"` + Draft bool `json:"draft"` + Author struct { + Login string `json:"login"` + ID int `json:"id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"author"` + Prerelease bool `json:"prerelease"` + CreatedAt time.Time `json:"created_at"` + PublishedAt time.Time `json:"published_at"` + Assets []struct { + URL string `json:"url"` + ID int `json:"id"` + Name string `json:"name"` + Label string `json:"label"` + Uploader struct { + Login string `json:"login"` + ID int `json:"id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"uploader"` + ContentType string `json:"content_type"` + State string `json:"state"` + Size int `json:"size"` + DownloadCount int `json:"download_count"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + BrowserDownloadURL string `json:"browser_download_url"` + } `json:"assets"` + TarballURL string `json:"tarball_url"` + ZipballURL string `json:"zipball_url"` + Body string `json:"body"` +} + +// checks if a path has write access +func writable(path string) bool { + return unix.Access(path, unix.W_OK) == nil +} + +// Directory to install releases in by default +// +// Find writable directories on $PATH. Use the first writable +// directory which is in $HOME or failing that the first writable +// directory. +// +// Returns "" if none of the above were found +func defaultBinDir() string { + home := os.Getenv("HOME") + var binDir string + for _, dir := range strings.Split(os.Getenv("PATH"), ":") { + if writable(dir) { + if strings.HasPrefix(dir, home) { + return dir + } + if binDir != "" { + binDir = dir + } + } + } + return binDir +} + +// read the body or an error message +func readBody(in io.Reader) string { + data, err := ioutil.ReadAll(in) + if err != nil { + return fmt.Sprintf("Error reading body: %v", err.Error()) + } + return string(data) +} + +// Get an asset URL and name +func getAsset(project string, matchName *regexp.Regexp) (string, string) { + url := "https://api.github.com/repos/" + project + "/releases/latest" + log.Printf("Fetching asset info for %q from %q", project, url) + user, pass := os.Getenv("GITHUB_USER"), os.Getenv("GITHUB_TOKEN") + req, err := http.NewRequest("GET", url, nil) + if err != nil { + log.Fatalf("Failed to make http request %q: %v", url, err) + } + if user != "" && pass != "" { + log.Printf("Fetching using GITHUB_USER and GITHUB_TOKEN") + req.SetBasicAuth(user, pass) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Fatalf("Failed to fetch release info %q: %v", url, err) + } + if resp.StatusCode != http.StatusOK { + log.Printf("Error: %s", readBody(resp.Body)) + log.Fatalf("Bad status %d when fetching %q release info: %s", resp.StatusCode, url, resp.Status) + } + var release Release + err = json.NewDecoder(resp.Body).Decode(&release) + if err != nil { + log.Fatalf("Failed to decode release info: %v", err) + } + err = resp.Body.Close() + if err != nil { + log.Fatalf("Failed to close body: %v", err) + } + + for _, asset := range release.Assets { + if matchName.MatchString(asset.Name) { + return asset.BrowserDownloadURL, asset.Name + } + } + log.Fatalf("Didn't find asset in info") + return "", "" +} + +// get a file for download +func getFile(url, fileName string) { + log.Printf("Downloading %q from %q", fileName, url) + + out, err := os.Create(fileName) + if err != nil { + log.Fatalf("Failed to open %q: %v", fileName, err) + } + + resp, err := http.Get(url) + if err != nil { + log.Fatalf("Failed to fetch asset %q: %v", url, err) + } + if resp.StatusCode != http.StatusOK { + log.Printf("Error: %s", readBody(resp.Body)) + log.Fatalf("Bad status %d when fetching %q asset: %s", resp.StatusCode, url, resp.Status) + } + + n, err := io.Copy(out, resp.Body) + if err != nil { + log.Fatalf("Error while downloading: %v", err) + } + + err = resp.Body.Close() + if err != nil { + log.Fatalf("Failed to close body: %v", err) + } + err = out.Close() + if err != nil { + log.Fatalf("Failed to close output file: %v", err) + } + + log.Printf("Downloaded %q (%d bytes)", fileName, n) +} + +// run a shell command +func run(args ...string) { + cmd := exec.Command(args[0], args[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + log.Fatalf("Failed to run %v: %v", args, err) + } +} + +func main() { + flag.Parse() + args := flag.Args() + if len(args) != 2 { + log.Fatalf("Syntax: %s ", os.Args[0]) + } + project, nameRe := args[0], args[1] + if !matchProject.MatchString(project) { + log.Fatalf("Project %q must be in form user/project", project) + } + matchName, err := regexp.Compile(nameRe) + if err != nil { + log.Fatalf("Invalid regexp for name %q: %v", nameRe, err) + } + + assetURL, assetName := getAsset(project, matchName) + fileName := filepath.Join(os.TempDir(), assetName) + getFile(assetURL, fileName) + + if *install { + log.Printf("Installing %s", fileName) + run("sudo", "dpkg", "--force-bad-version", "-i", fileName) + log.Printf("Installed %s", fileName) + } else if *extract != "" { + if *bindir == "" { + log.Fatalf("Need to set -bindir") + } + log.Printf("Unpacking %s from %s and installing into %s", *extract, fileName, *bindir) + run("tar", "xf", fileName, *extract) + run("chmod", "a+x", *extract) + run("mv", "-f", *extract, *bindir+"/") + } +} diff --git a/.rclone_repo/bin/make_changelog.py b/.rclone_repo/bin/make_changelog.py new file mode 100755 index 0000000..0c18d68 --- /dev/null +++ b/.rclone_repo/bin/make_changelog.py @@ -0,0 +1,173 @@ +#!/usr/bin/python +""" +Generate a markdown changelog for the rclone project +""" + +import os +import sys +import re +import datetime +import subprocess +from collections import defaultdict + +IGNORE_RES = [ + r"^Add .* to contributors$", + r"^Start v\d+.\d+-DEV development$", + r"^Version v\d.\d+$", +] + +IGNORE_RE = re.compile("(?:" + "|".join(IGNORE_RES) + ")") + +CATEGORY = re.compile(r"(^[\w/ ]+(?:, *[\w/ ]+)*):\s*(.*)$") + +backends = [ x for x in os.listdir("backend") if x != "all"] + +backend_aliases = { + "amazon cloud drive" : "amazonclouddrive", + "acd" : "amazonclouddrive", + "google cloud storage" : "googlecloudstorage", + "gcs" : "googlecloudstorage", + "azblob" : "azureblob", + "mountlib": "mount", + "cmount": "mount", + "mount/cmount": "mount", +} + +backend_titles = { + "amazonclouddrive": "Amazon Cloud Drive", + "googlecloudstorage": "Google Cloud Storage", + "azureblob": "Azure Blob", + "ftp": "FTP", + "sftp": "SFTP", + "http": "HTTP", + "webdav": "WebDAV", +} + +STRIP_FIX_RE = re.compile(r"(\s+-)?\s+((fixes|addresses)\s+)?#\d+", flags=re.I) + +STRIP_PATH_RE = re.compile(r"^(backend|fs)/") + +IS_FIX_RE = re.compile(r"\b(fix|fixes)\b", flags=re.I) + +def make_out(data, indent=""): + """Return a out, lines the first being a function for output into the second""" + out_lines = [] + def out(category, title=None): + if title == None: + title = category + lines = data.get(category) + if not lines: + return + del(data[category]) + if indent != "" and len(lines) == 1: + out_lines.append(indent+"* " + title+": " + lines[0]) + return + out_lines.append(indent+"* " + title) + for line in lines: + out_lines.append(indent+" * " + line) + return out, out_lines + + +def process_log(log): + """Process the incoming log into a category dict of lists""" + by_category = defaultdict(list) + for log_line in reversed(log.split("\n")): + log_line = log_line.strip() + hash, author, timestamp, message = log_line.split("|", 3) + message = message.strip() + if IGNORE_RE.search(message): + continue + match = CATEGORY.search(message) + categories = "UNKNOWN" + if match: + categories = match.group(1).lower() + message = match.group(2) + message = STRIP_FIX_RE.sub("", message) + message = message +" ("+author+")" + message = message[0].upper()+message[1:] + seen = set() + for category in categories.split(","): + category = category.strip() + category = STRIP_PATH_RE.sub("", category) + category = backend_aliases.get(category, category) + if category in seen: + continue + by_category[category].append(message) + seen.add(category) + #print category, hash, author, timestamp, message + return by_category + +def main(): + if len(sys.argv) != 3: + print >>sys.stderr, "Syntax: %s vX.XX vX.XY" % sys.argv[0] + sys.exit(1) + version, next_version = sys.argv[1], sys.argv[2] + log = subprocess.check_output(["git", "log", '''--pretty=format:%H|%an|%aI|%s'''] + [version+".."+next_version]) + by_category = process_log(log) + + # Output backends first so remaining in by_category are core items + out, backend_lines = make_out(by_category) + out("mount", title="Mount") + out("vfs", title="VFS") + out("local", title="Local") + out("cache", title="Cache") + out("crypt", title="Crypt") + backend_names = sorted(x for x in by_category.keys() if x in backends) + for backend_name in backend_names: + if backend_name in backend_titles: + backend_title = backend_titles[backend_name] + else: + backend_title = backend_name.title() + out(backend_name, title=backend_title) + + # Split remaining in by_category into new features and fixes + new_features = defaultdict(list) + bugfixes = defaultdict(list) + for name, messages in by_category.iteritems(): + for message in messages: + if IS_FIX_RE.search(message): + bugfixes[name].append(message) + else: + new_features[name].append(message) + + # Output new features + out, new_features_lines = make_out(new_features, indent=" ") + for name in sorted(new_features.keys()): + out(name) + + # Output bugfixes + out, bugfix_lines = make_out(bugfixes, indent=" ") + for name in sorted(bugfixes.keys()): + out(name) + + # Read old changlog and split + with open("docs/content/changelog.md") as fd: + old_changelog = fd.read() + heading = "# Changelog" + i = old_changelog.find(heading) + if i < 0: + raise AssertionError("Couldn't find heading in old changelog") + i += len(heading) + old_head, old_tail = old_changelog[:i], old_changelog[i:] + + # Update the build date + old_head = re.sub(r"\d\d\d\d-\d\d-\d\d", str(datetime.date.today()), old_head) + + # Output combined changelog with new part + sys.stdout.write(old_head) + sys.stdout.write(""" + +## %s - %s + +* New backends +* New commands +* New Features +%s +* Bug Fixes +%s +%s""" % (version, datetime.date.today(), "\n".join(new_features_lines), "\n".join(bugfix_lines), "\n".join(backend_lines))) + sys.stdout.write(old_tail) + + +if __name__ == "__main__": + main() diff --git a/.rclone_repo/bin/make_manual.py b/.rclone_repo/bin/make_manual.py new file mode 100755 index 0000000..68ab745 --- /dev/null +++ b/.rclone_repo/bin/make_manual.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +""" +Make single page versions of the documentation for release and +conversion into man pages etc. +""" + +import os +import re +from datetime import datetime + +docpath = "docs/content" +outfile = "MANUAL.md" + +# Order to add docs segments to make outfile +docs = [ + "about.md", + "install.md", + "docs.md", + "remote_setup.md", + "filtering.md", + "rc.md", + "overview.md", + + # Keep these alphabetical by full name + "alias.md", + "amazonclouddrive.md", + "s3.md", + "b2.md", + "box.md", + "cache.md", + "crypt.md", + "dropbox.md", + "ftp.md", + "googlecloudstorage.md", + "drive.md", + "http.md", + "hubic.md", + "jottacloud.md", + "mega.md", + "azureblob.md", + "onedrive.md", + "opendrive.md", + "qingstor.md", + "swift.md", + "pcloud.md", + "sftp.md", + "webdav.md", + "yandex.md", + + "local.md", + "changelog.md", + "bugs.md", + "faq.md", + "licence.md", + "authors.md", + "contact.md", +] + +# Order to put the commands in - any not on here will be in sorted order +commands_order = [ + "rclone_config.md", + "rclone_copy.md", + "rclone_sync.md", + "rclone_move.md", + "rclone_delete.md", + "rclone_purge.md", + "rclone_mkdir.md", + "rclone_rmdir.md", + "rclone_check.md", + "rclone_ls.md", + "rclone_lsd.md", + "rclone_lsl.md", + "rclone_md5sum.md", + "rclone_sha1sum.md", + "rclone_size.md", + "rclone_version.md", + "rclone_cleanup.md", + "rclone_dedupe.md", +] + +# Docs which aren't made into outfile +ignore_docs = [ + "downloads.md", + "privacy.md", + "donate.md", +] + +def read_doc(doc): + """Read file as a string""" + path = os.path.join(docpath, doc) + with open(path) as fd: + contents = fd.read() + parts = contents.split("---\n", 2) + if len(parts) != 3: + raise ValueError("Couldn't find --- markers: found %d parts" % len(parts)) + contents = parts[2].strip()+"\n\n" + # Remove icons + contents = re.sub(r'}} + contents = re.sub(r'\{\{<\s+provider.*?name="(.*?)".*?>\}\}', r"\1", contents) + return contents + +def check_docs(docpath): + """Check all the docs are in docpath""" + files = set(f for f in os.listdir(docpath) if f.endswith(".md")) + files -= set(ignore_docs) + docs_set = set(docs) + if files == docs_set: + return + print "Files on disk but not in docs variable: %s" % ", ".join(files - docs_set) + print "Files in docs variable but not on disk: %s" % ", ".join(docs_set - files) + raise ValueError("Missing files") + +def read_command(command): + doc = read_doc("commands/"+command) + doc = re.sub(r"### Options inherited from parent commands.*$", "", doc, 0, re.S) + doc = doc.strip()+"\n" + return doc + +def read_commands(docpath): + """Reads the commands an makes them into a single page""" + files = set(f for f in os.listdir(docpath + "/commands") if f.endswith(".md")) + docs = [] + for command in commands_order: + docs.append(read_command(command)) + files.remove(command) + for command in sorted(files): + if command != "rclone.md": + docs.append(read_command(command)) + return "\n".join(docs) + +def main(): + check_docs(docpath) + command_docs = read_commands(docpath) + with open(outfile, "w") as out: + out.write("""\ +%% rclone(1) User Manual +%% Nick Craig-Wood +%% %s + +""" % datetime.now().strftime("%b %d, %Y")) + for doc in docs: + contents = read_doc(doc) + # Substitute the commands into doc.md + if doc == "docs.md": + contents = re.sub(r"The main rclone commands.*?for the full list.", command_docs, contents, 0, re.S) + out.write(contents) + print "Written '%s'" % outfile + +if __name__ == "__main__": + main() diff --git a/.rclone_repo/bin/make_rc_docs.sh b/.rclone_repo/bin/make_rc_docs.sh new file mode 100755 index 0000000..1f2c3e1 --- /dev/null +++ b/.rclone_repo/bin/make_rc_docs.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Insert the rc docs into docs/content/rc.md + +set -e + +go install +mkdir -p /tmp/rclone_cache_test +export RCLONE_CONFIG_RCDOCS_TYPE=cache +export RCLONE_CONFIG_RCDOCS_REMOTE=/tmp/rclone/cache_test +rclone -q --rc mount rcdocs: /mnt/tmp/ & +sleep 0.5 +rclone rc > /tmp/z.md +fusermount -z -u /mnt/tmp/ + +awk ' + BEGIN {p=1} + /^ +### cache/expire: Purge a remote from cache + +Purge a remote from the cache backend. Supports either a directory or a file. +Params: + - remote = path to remote (required) + - withData = true/false to delete cached data (chunks) as well (optional) + +Eg + + rclone rc cache/expire remote=path/to/sub/folder/ + rclone rc cache/expire remote=/ withData=true + +### cache/stats: Get cache stats + +Show statistics for the cache remote. + +### core/bwlimit: Set the bandwidth limit. + +This sets the bandwidth limit to that passed in. + +Eg + + rclone rc core/bwlimit rate=1M + rclone rc core/bwlimit rate=off + +The format of the parameter is exactly the same as passed to --bwlimit +except only one bandwidth may be specified. + +### core/gc: Runs a garbage collection. + +This tells the go runtime to do a garbage collection run. It isn't +necessary to call this normally, but it can be useful for debugging +memory problems. + +### core/memstats: Returns the memory statistics + +This returns the memory statistics of the running program. What the values mean +are explained in the go docs: https://golang.org/pkg/runtime/#MemStats + +The most interesting values for most people are: + +* HeapAlloc: This is the amount of memory rclone is actually using +* HeapSys: This is the amount of memory rclone has obtained from the OS +* Sys: this is the total amount of memory requested from the OS + * It is virtual memory so may include unused memory + +### core/pid: Return PID of current process + +This returns PID of current process. +Useful for stopping rclone process. + +### core/stats: Returns stats about current transfers. + +This returns all available stats + + rclone rc core/stats + +Returns the following values: + +``` +{ + "speed": average speed in bytes/sec since start of the process, + "bytes": total transferred bytes since the start of the process, + "errors": number of errors, + "checks": number of checked files, + "transfers": number of transferred files, + "deletes" : number of deleted files, + "elapsedTime": time in seconds since the start of the process, + "lastError": last occurred error, + "transferring": an array of currently active file transfers: + [ + { + "bytes": total transferred bytes for this file, + "eta": estimated time in seconds until file transfer completion + "name": name of the file, + "percentage": progress of the file transfer in percent, + "speed": speed in bytes/sec, + "speedAvg": speed in bytes/sec as an exponentially weighted moving average, + "size": size of the file in bytes + } + ], + "checking": an array of names of currently active file checks + [] +} +``` +Values for "transferring", "checking" and "lastError" are only assigned if data is available. +The value for "eta" is null if an eta cannot be determined. + +### rc/error: This returns an error + +This returns an error with the input as part of its error string. +Useful for testing error handling. + +### rc/list: List all the registered remote control commands + +This lists all the registered remote control commands as a JSON map in +the commands response. + +### rc/noop: Echo the input to the output parameters + +This echoes the input parameters to the output parameters for testing +purposes. It can be used to check that rclone is still alive and to +check that parameter passing is working properly. + +### vfs/forget: Forget files or directories in the directory cache. + +This forgets the paths in the directory cache causing them to be +re-read from the remote when needed. + +If no paths are passed in then it will forget all the paths in the +directory cache. + + rclone rc vfs/forget + +Otherwise pass files or dirs in as file=path or dir=path. Any +parameter key starting with file will forget that file and any +starting with dir will forget that dir, eg + + rclone rc vfs/forget file=hello file2=goodbye dir=home/junk + +### vfs/refresh: Refresh the directory cache. + +This reads the directories for the specified paths and freshens the +directory cache. + +If no paths are passed in then it will refresh the root directory. + + rclone rc vfs/refresh + +Otherwise pass directories in as dir=path. Any parameter key +starting with dir will refresh that directory, eg + + rclone rc vfs/refresh dir=home/junk dir2=data/misc + +If the parameter recursive=true is given the whole directory tree +will get refreshed. This refresh will use --fast-list if enabled. + + + +## Accessing the remote control via HTTP + +Rclone implements a simple HTTP based protocol. + +Each endpoint takes an JSON object and returns a JSON object or an +error. The JSON objects are essentially a map of string names to +values. + +All calls must made using POST. + +The input objects can be supplied using URL parameters, POST +parameters or by supplying "Content-Type: application/json" and a JSON +blob in the body. There are examples of these below using `curl`. + +The response will be a JSON blob in the body of the response. This is +formatted to be reasonably human readable. + +If an error occurs then there will be an HTTP error status (usually +400) and the body of the response will contain a JSON encoded error +object. + +### Using POST with URL parameters only + +``` +curl -X POST 'http://localhost:5572/rc/noop/?potato=1&sausage=2' +``` + +Response + +``` +{ + "potato": "1", + "sausage": "2" +} +``` + +Here is what an error response looks like: + +``` +curl -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2' +``` + +``` +{ + "error": "arbitrary error on input map[potato:1 sausage:2]", + "input": { + "potato": "1", + "sausage": "2" + } +} +``` + +Note that curl doesn't return errors to the shell unless you use the `-f` option + +``` +$ curl -f -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2' +curl: (22) The requested URL returned error: 400 Bad Request +$ echo $? +22 +``` + +### Using POST with a form + +``` +curl --data "potato=1" --data "sausage=2" http://localhost:5572/rc/noop/ +``` + +Response + +``` +{ + "potato": "1", + "sausage": "2" +} +``` + +Note that you can combine these with URL parameters too with the POST +parameters taking precedence. + +``` +curl --data "potato=1" --data "sausage=2" "http://localhost:5572/rc/noop/?rutabaga=3&sausage=4" +``` + +Response + +``` +{ + "potato": "1", + "rutabaga": "3", + "sausage": "4" +} + +``` + +### Using POST with a JSON blob + +``` +curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' http://localhost:5572/rc/noop/ +``` + +response + +``` +{ + "password": "xyz", + "username": "xyz" +} +``` + +This can be combined with URL parameters too if required. The JSON +blob takes precedence. + +``` +curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' 'http://localhost:5572/rc/noop/?rutabaga=3&potato=4' +``` + +``` +{ + "potato": 2, + "rutabaga": "3", + "sausage": 1 +} +``` + +## Debugging rclone with pprof ## + +If you use the `--rc` flag this will also enable the use of the go +profiling tools on the same port. + +To use these, first [install go](https://golang.org/doc/install). + +Then (for example) to profile rclone's memory use you can run: + + go tool pprof -web http://localhost:5572/debug/pprof/heap + +This should open a page in your browser showing what is using what +memory. + +You can also use the `-text` flag to produce a textual summary + +``` +$ go tool pprof -text http://localhost:5572/debug/pprof/heap +Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total + flat flat% sum% cum cum% + 1024.03kB 66.62% 66.62% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.addDecoderNode + 513kB 33.38% 100% 513kB 33.38% net/http.newBufioWriterSize + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/cmd/all.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/cmd/serve.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/cmd/serve/restic.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init + 0 0% 100% 1024.03kB 66.62% github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init.0 + 0 0% 100% 1024.03kB 66.62% main.init + 0 0% 100% 513kB 33.38% net/http.(*conn).readRequest + 0 0% 100% 513kB 33.38% net/http.(*conn).serve + 0 0% 100% 1024.03kB 66.62% runtime.main +``` + +Possible profiles to look at: + + * Memory: `go tool pprof http://localhost:5572/debug/pprof/heap` + * 30-second CPU profile: `go tool pprof http://localhost:5572/debug/pprof/profile` + * 5-second execution trace: `wget http://localhost:5572/debug/pprof/trace?seconds=5` + +See the [net/http/pprof docs](https://golang.org/pkg/net/http/pprof/) +for more info on how to use the profiling and for a general overview +see [the Go team's blog post on profiling go programs](https://blog.golang.org/profiling-go-programs). + +The profiling hook is [zero overhead unless it is used](https://stackoverflow.com/q/26545159/164234). diff --git a/.rclone_repo/docs/content/remote_setup.md b/.rclone_repo/docs/content/remote_setup.md new file mode 100755 index 0000000..67de108 --- /dev/null +++ b/.rclone_repo/docs/content/remote_setup.md @@ -0,0 +1,88 @@ +--- +title: "Remote Setup" +description: "Configuring rclone up on a remote / headless machine" +date: "2016-01-07" +--- + +# Configuring rclone on a remote / headless machine # + +Some of the configurations (those involving oauth2) require an +Internet connected web browser. + +If you are trying to set rclone up on a remote or headless box with no +browser available on it (eg a NAS or a server in a datacenter) then +you will need to use an alternative means of configuration. There are +two ways of doing it, described below. + +## Configuring using rclone authorize ## + +On the headless box + +``` +... +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> n +For this to work, you will need rclone available on a machine that has a web browser available. +Execute the following on your machine: + rclone authorize "amazon cloud drive" +Then paste the result below: +result> +``` + +Then on your main desktop machine + +``` +rclone authorize "amazon cloud drive" +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +Paste the following into your remote machine ---> +SECRET_TOKEN +<---End paste +``` + +Then back to the headless box, paste in the code + +``` +result> SECRET_TOKEN +-------------------- +[acd12] +client_id = +client_secret = +token = SECRET_TOKEN +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> +``` + +## Configuring by copying the config file ## + +Rclone stores all of its config in a single configuration file. This +can easily be copied to configure a remote rclone. + +So first configure rclone on your desktop machine + + rclone config + +to set up the config file. + +Find the config file by running `rclone -h` and looking for the help for the `--config` option + +``` +$ rclone -h +[snip] + --config="/home/user/.rclone.conf": Config file. +[snip] +``` + +Now transfer it to the remote box (scp, cut paste, ftp, sftp etc) and +place it in the correct place (use `rclone -h` on the remote box to +find out where). diff --git a/.rclone_repo/docs/content/s3.md b/.rclone_repo/docs/content/s3.md new file mode 100755 index 0000000..8732401 --- /dev/null +++ b/.rclone_repo/docs/content/s3.md @@ -0,0 +1,1027 @@ +--- +title: "Amazon S3" +description: "Rclone docs for Amazon S3" +date: "2016-07-11" +--- + + Amazon S3 Storage Providers +-------------------------------------------------------- + +The S3 backend can be used with a number of different providers: + +* {{< provider name="AWS S3" home="https://aws.amazon.com/s3/" config="/s3/#amazon-s3" >}} +* {{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}} +* {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}} +* {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} +* {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} +* {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}} +* {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}} + +Paths are specified as `remote:bucket` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, eg `remote:bucket/path/to/dir`. + +Once you have made a remote (see the provider specific section above) +you can use it like this: + +See all buckets + + rclone lsd remote: + +Make a new bucket + + rclone mkdir remote:bucket + +List the contents of a bucket + + rclone ls remote:bucket + +Sync `/home/local/directory` to the remote bucket, deleting any excess +files in the bucket. + + rclone sync /home/local/directory remote:bucket + +## AWS S3 {#amazon-s3} + +Here is an example of making an s3 configuration. First run + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" + 2 / Amazon Drive + \ "amazon cloud drive" + 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio) + \ "s3" + 4 / Backblaze B2 + \ "b2" +[snip] +23 / http Connection + \ "http" +Storage> s3 +Choose your S3 provider. +Choose a number from below, or type in your own value + 1 / Amazon Web Services (AWS) S3 + \ "AWS" + 2 / Ceph Object Storage + \ "Ceph" + 3 / Digital Ocean Spaces + \ "DigitalOcean" + 4 / Dreamhost DreamObjects + \ "Dreamhost" + 5 / IBM COS S3 + \ "IBMCOS" + 6 / Minio Object Storage + \ "Minio" + 7 / Wasabi Object Storage + \ "Wasabi" + 8 / Any other S3 compatible provider + \ "Other" +provider> 1 +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" +env_auth> 1 +AWS Access Key ID - leave blank for anonymous access or runtime credentials. +access_key_id> XXX +AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. +secret_access_key> YYY +Region to connect to. +Choose a number from below, or type in your own value + / The default endpoint - a good choice if you are unsure. + 1 | US Region, Northern Virginia or Pacific Northwest. + | Leave location constraint empty. + \ "us-east-1" + / US East (Ohio) Region + 2 | Needs location constraint us-east-2. + \ "us-east-2" + / US West (Oregon) Region + 3 | Needs location constraint us-west-2. + \ "us-west-2" + / US West (Northern California) Region + 4 | Needs location constraint us-west-1. + \ "us-west-1" + / Canada (Central) Region + 5 | Needs location constraint ca-central-1. + \ "ca-central-1" + / EU (Ireland) Region + 6 | Needs location constraint EU or eu-west-1. + \ "eu-west-1" + / EU (London) Region + 7 | Needs location constraint eu-west-2. + \ "eu-west-2" + / EU (Frankfurt) Region + 8 | Needs location constraint eu-central-1. + \ "eu-central-1" + / Asia Pacific (Singapore) Region + 9 | Needs location constraint ap-southeast-1. + \ "ap-southeast-1" + / Asia Pacific (Sydney) Region +10 | Needs location constraint ap-southeast-2. + \ "ap-southeast-2" + / Asia Pacific (Tokyo) Region +11 | Needs location constraint ap-northeast-1. + \ "ap-northeast-1" + / Asia Pacific (Seoul) +12 | Needs location constraint ap-northeast-2. + \ "ap-northeast-2" + / Asia Pacific (Mumbai) +13 | Needs location constraint ap-south-1. + \ "ap-south-1" + / South America (Sao Paulo) Region +14 | Needs location constraint sa-east-1. + \ "sa-east-1" +region> 1 +Endpoint for S3 API. +Leave blank if using AWS to use the default endpoint for the region. +endpoint> +Location constraint - must be set to match the Region. Used when creating buckets only. +Choose a number from below, or type in your own value + 1 / Empty for US Region, Northern Virginia or Pacific Northwest. + \ "" + 2 / US East (Ohio) Region. + \ "us-east-2" + 3 / US West (Oregon) Region. + \ "us-west-2" + 4 / US West (Northern California) Region. + \ "us-west-1" + 5 / Canada (Central) Region. + \ "ca-central-1" + 6 / EU (Ireland) Region. + \ "eu-west-1" + 7 / EU (London) Region. + \ "eu-west-2" + 8 / EU Region. + \ "EU" + 9 / Asia Pacific (Singapore) Region. + \ "ap-southeast-1" +10 / Asia Pacific (Sydney) Region. + \ "ap-southeast-2" +11 / Asia Pacific (Tokyo) Region. + \ "ap-northeast-1" +12 / Asia Pacific (Seoul) + \ "ap-northeast-2" +13 / Asia Pacific (Mumbai) + \ "ap-south-1" +14 / South America (Sao Paulo) Region. + \ "sa-east-1" +location_constraint> 1 +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" + 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access. + \ "public-read" + / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. + 3 | Granting this on a bucket is generally not recommended. + \ "public-read-write" + 4 / Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. + \ "authenticated-read" + / Object owner gets FULL_CONTROL. Bucket owner gets READ access. + 5 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ "bucket-owner-read" + / Both the object owner and the bucket owner get FULL_CONTROL over the object. + 6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ "bucket-owner-full-control" +acl> 1 +The server-side encryption algorithm used when storing this object in S3. +Choose a number from below, or type in your own value + 1 / None + \ "" + 2 / AES256 + \ "AES256" +server_side_encryption> 1 +The storage class to use when storing objects in S3. +Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Standard storage class + \ "STANDARD" + 3 / Reduced redundancy storage class + \ "REDUCED_REDUNDANCY" + 4 / Standard Infrequent Access storage class + \ "STANDARD_IA" + 5 / One Zone Infrequent Access storage class + \ "ONEZONE_IA" +storage_class> 1 +Remote config +-------------------- +[remote] +type = s3 +provider = AWS +env_auth = false +access_key_id = XXX +secret_access_key = YYY +region = us-east-1 +endpoint = +location_constraint = +acl = private +server_side_encryption = +storage_class = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> +``` + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +### --update and --use-server-modtime ### + +As noted below, the modified time is stored on metadata on the object. It is +used by default for all operations that require checking the time a file was +last updated. It allows rclone to treat the remote more like a true filesystem, +but it is inefficient because it requires an extra API call to retrieve the +metadata. + +For many operations, the time the object was last uploaded to the remote is +sufficient to determine if it is "dirty". By using `--update` along with +`--use-server-modtime`, you can avoid the extra API call and simply upload +files whose local modtime is newer than the time it was last uploaded. + +### Modified time ### + +The modified time is stored as metadata on the object as +`X-Amz-Meta-Mtime` as floating point since the epoch accurate to 1 ns. + +### Multipart uploads ### + +rclone supports multipart uploads with S3 which means that it can +upload files bigger than 5GB. Note that files uploaded *both* with +multipart upload *and* through crypt remotes do not have MD5 sums. + +### Buckets and Regions ### + +With Amazon S3 you can list buckets (`rclone lsd`) using any region, +but you can only access the content of a bucket from the region it was +created in. If you attempt to access a bucket from the wrong region, +you will get an error, `incorrect region, the bucket is not in 'XXX' +region`. + +### Authentication ### + +There are a number of ways to supply `rclone` with a set of AWS +credentials, with and without using the environment. + +The different authentication methods are tried in this order: + + - Directly in the rclone configuration file (`env_auth = false` in the config file): + - `access_key_id` and `secret_access_key` are required. + - `session_token` can be optionally set when using AWS STS. + - Runtime configuration (`env_auth = true` in the config file): + - Export the following environment variables before running `rclone`: + - Access Key ID: `AWS_ACCESS_KEY_ID` or `AWS_ACCESS_KEY` + - Secret Access Key: `AWS_SECRET_ACCESS_KEY` or `AWS_SECRET_KEY` + - Session Token: `AWS_SESSION_TOKEN` (optional) + - Or, use a [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-multiple-profiles.html): + - Profile files are standard files used by AWS CLI tools + - By default it will use the profile in your home directory (eg `~/.aws/credentials` on unix based systems) file and the "default" profile, to change set these environment variables: + - `AWS_SHARED_CREDENTIALS_FILE` to control which file. + - `AWS_PROFILE` to control which profile to use. + - Or, run `rclone` in an ECS task with an IAM role (AWS only). + - Or, run `rclone` on an EC2 instance with an IAM role (AWS only). + +If none of these option actually end up providing `rclone` with AWS +credentials then S3 interaction will be non-authenticated (see below). + +### S3 Permissions ### + +When using the `sync` subcommand of `rclone` the following minimum +permissions are required to be available on the bucket being written to: + +* `ListBucket` +* `DeleteObject` +* `GetObject` +* `PutObject` +* `PutObjectACL` + +Example policy: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::USER_SID:user/USER_NAME" + }, + "Action": [ + "s3:ListBucket", + "s3:DeleteObject", + "s3:GetObject", + "s3:PutObject", + "s3:PutObjectAcl" + ], + "Resource": [ + "arn:aws:s3:::BUCKET_NAME/*", + "arn:aws:s3:::BUCKET_NAME" + ] + } + ] +} +``` + +Notes on above: + +1. This is a policy that can be used when creating bucket. It assumes + that `USER_NAME` has been created. +2. The Resource entry must include both resource ARNs, as one implies + the bucket and the other implies the bucket's objects. + +For reference, [here's an Ansible script](https://gist.github.com/ebridges/ebfc9042dd7c756cd101cfa807b7ae2b) +that will generate one or more buckets that will work with `rclone sync`. + +### Key Management System (KMS) ### + +If you are using server side encryption with KMS then you will find +you can't transfer small objects. As a work-around you can use the +`--ignore-checksum` flag. + +A proper fix is being worked on in [issue #1824](https://github.com/ncw/rclone/issues/1824). + +### Glacier ### + +You can transition objects to glacier storage using a [lifecycle policy](http://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-lifecycle.html). +The bucket can still be synced or copied into normally, but if rclone +tries to access the data you will see an error like below. + + 2017/09/11 19:07:43 Failed to sync: failed to open source object: Object in GLACIER, restore first: path/to/file + +In this case you need to [restore](http://docs.aws.amazon.com/AmazonS3/latest/user-guide/restore-archived-objects.html) +the object(s) in question before using rclone. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --s3-acl=STRING #### + +Canned ACL used when creating buckets and/or storing objects in S3. + +For more info visit the [canned ACL docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl). + +#### --s3-storage-class=STRING #### + +Storage class to upload new objects with. + +Available options include: + + - STANDARD - default storage class + - STANDARD_IA - for less frequently accessed data (e.g backups) + - ONEZONE_IA - for storing data in only one Availability Zone + - REDUCED_REDUNDANCY (only for noncritical, reproducible data, has lower redundancy) + +#### --s3-chunk-size=SIZE #### + +Any files larger than this will be uploaded in chunks of this +size. The default is 5MB. The minimum is 5MB. + +Note that 2 chunks of this size are buffered in memory per transfer. + +If you are transferring large files over high speed links and you have +enough memory, then increasing this will speed up the transfers. + +#### --s3-force-path-style=BOOL #### + +If this is true (the default) then rclone will use path style access, +if false then rclone will use virtual path style. See [the AWS S3 +docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro) +for more info. + +Some providers (eg Aliyun OSS or Netease COS) require this set to +`false`. It can also be set in the config in the advanced section. + +#### --s3-upload-concurrency #### + +Number of chunks of the same file that are uploaded concurrently. +Default is 2. + +If you are uploading small amount of large file over high speed link +and these uploads do not fully utilize your bandwidth, then increasing +this may help to speed up the transfers. + +### Anonymous access to public buckets ### + +If you want to use rclone to access a public bucket, configure with a +blank `access_key_id` and `secret_access_key`. Your config should end +up looking like this: + +``` +[anons3] +type = s3 +provider = AWS +env_auth = false +access_key_id = +secret_access_key = +region = us-east-1 +endpoint = +location_constraint = +acl = private +server_side_encryption = +storage_class = +``` + +Then use it as normal with the name of the public bucket, eg + + rclone lsd anons3:1000genomes + +You will be able to list and copy data but not upload it. + +### Ceph ### + +[Ceph](https://ceph.com/) is an open source unified, distributed +storage system designed for excellent performance, reliability and +scalability. It has an S3 compatible object storage interface. + +To use rclone with Ceph, configure as above but leave the region blank +and set the endpoint. You should end up with something like this in +your config: + + +``` +[ceph] +type = s3 +provider = Ceph +env_auth = false +access_key_id = XXX +secret_access_key = YYY +region = +endpoint = https://ceph.endpoint.example.com +location_constraint = +acl = +server_side_encryption = +storage_class = +``` + +Note also that Ceph sometimes puts `/` in the passwords it gives +users. If you read the secret access key using the command line tools +you will get a JSON blob with the `/` escaped as `\/`. Make sure you +only write `/` in the secret access key. + +Eg the dump from Ceph looks something like this (irrelevant keys +removed). + +``` +{ + "user_id": "xxx", + "display_name": "xxxx", + "keys": [ + { + "user": "xxx", + "access_key": "xxxxxx", + "secret_key": "xxxxxx\/xxxx" + } + ], +} +``` + +Because this is a json dump, it is encoding the `/` as `\/`, so if you +use the secret key as `xxxxxx/xxxx` it will work fine. + +### Dreamhost ### + +Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is +an object storage system based on CEPH. + +To use rclone with Dreamhost, configure as above but leave the region blank +and set the endpoint. You should end up with something like this in +your config: + +``` +[dreamobjects] +type = s3 +provider = DreamHost +env_auth = false +access_key_id = your_access_key +secret_access_key = your_secret_key +region = +endpoint = objects-us-west-1.dream.io +location_constraint = +acl = private +server_side_encryption = +storage_class = +``` + +### DigitalOcean Spaces ### + +[Spaces](https://www.digitalocean.com/products/object-storage/) is an [S3-interoperable](https://developers.digitalocean.com/documentation/spaces/) object storage service from cloud provider DigitalOcean. + +To connect to DigitalOcean Spaces you will need an access key and secret key. These can be retrieved on the "[Applications & API](https://cloud.digitalocean.com/settings/api/tokens)" page of the DigitalOcean control panel. They will be needed when promted by `rclone config` for your `access_key_id` and `secret_access_key`. + +When prompted for a `region` or `location_constraint`, press enter to use the default value. The region must be included in the `endpoint` setting (e.g. `nyc3.digitaloceanspaces.com`). The defualt values can be used for other settings. + +Going through the whole process of creating a new remote by running `rclone config`, each prompt should be answered as shown below: + +``` +Storage> s3 +env_auth> 1 +access_key_id> YOUR_ACCESS_KEY +secret_access_key> YOUR_SECRET_KEY +region> +endpoint> nyc3.digitaloceanspaces.com +location_constraint> +acl> +storage_class> +``` + +The resulting configuration file should look like: + +``` +[spaces] +type = s3 +provider = DigitalOcean +env_auth = false +access_key_id = YOUR_ACCESS_KEY +secret_access_key = YOUR_SECRET_KEY +region = +endpoint = nyc3.digitaloceanspaces.com +location_constraint = +acl = +server_side_encryption = +storage_class = +``` + +Once configured, you can create a new Space and begin copying files. For example: + +``` +rclone mkdir spaces:my-new-space +rclone copy /path/to/files spaces:my-new-space +``` + +### IBM COS (S3) ### + +Information stored with IBM Cloud Object Storage is encrypted and dispersed across multiple geographic locations, and accessed through an implementation of the S3 API. This service makes use of the distributed storage technologies provided by IBM’s Cloud Object Storage System (formerly Cleversafe). For more information visit: (http://www.ibm.com/cloud/object-storage) + +To configure access to IBM COS S3, follow the steps below: + +1. Run rclone config and select n for a new remote. +``` + 2018/02/14 14:13:11 NOTICE: Config file "C:\\Users\\a\\.config\\rclone\\rclone.conf" not found - using defaults + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n +``` + +2. Enter the name for the configuration +``` + name> +``` + +3. Select "s3" storage. +``` +Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" + 2 / Amazon Drive + \ "amazon cloud drive" + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, Minio, IBM COS) + \ "s3" + 4 / Backblaze B2 + \ "b2" +[snip] + 23 / http Connection + \ "http" +Storage> 3 +``` + +4. Select IBM COS as the S3 Storage Provider. +``` +Choose the S3 provider. +Choose a number from below, or type in your own value + 1 / Choose this option to configure Storage to AWS S3 + \ "AWS" + 2 / Choose this option to configure Storage to Ceph Systems + \ "Ceph" + 3 / Choose this option to configure Storage to Dreamhost + \ "Dreamhost" + 4 / Choose this option to the configure Storage to IBM COS S3 + \ "IBMCOS" + 5 / Choose this option to the configure Storage to Minio + \ "Minio" + Provider>4 +``` + +5. Enter the Access Key and Secret. +``` + AWS Access Key ID - leave blank for anonymous access or runtime credentials. + access_key_id> <> + AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. + secret_access_key> <> +``` + +6. Specify the endpoint for IBM COS. For Public IBM COS, choose from the option below. For On Premise IBM COS, enter an enpoint address. +``` + Endpoint for IBM COS S3 API. + Specify if using an IBM COS On Premise. + Choose a number from below, or type in your own value + 1 / US Cross Region Endpoint + \ "s3-api.us-geo.objectstorage.softlayer.net" + 2 / US Cross Region Dallas Endpoint + \ "s3-api.dal.us-geo.objectstorage.softlayer.net" + 3 / US Cross Region Washington DC Endpoint + \ "s3-api.wdc-us-geo.objectstorage.softlayer.net" + 4 / US Cross Region San Jose Endpoint + \ "s3-api.sjc-us-geo.objectstorage.softlayer.net" + 5 / US Cross Region Private Endpoint + \ "s3-api.us-geo.objectstorage.service.networklayer.com" + 6 / US Cross Region Dallas Private Endpoint + \ "s3-api.dal-us-geo.objectstorage.service.networklayer.com" + 7 / US Cross Region Washington DC Private Endpoint + \ "s3-api.wdc-us-geo.objectstorage.service.networklayer.com" + 8 / US Cross Region San Jose Private Endpoint + \ "s3-api.sjc-us-geo.objectstorage.service.networklayer.com" + 9 / US Region East Endpoint + \ "s3.us-east.objectstorage.softlayer.net" + 10 / US Region East Private Endpoint + \ "s3.us-east.objectstorage.service.networklayer.com" + 11 / US Region South Endpoint +[snip] + 34 / Toronto Single Site Private Endpoint + \ "s3.tor01.objectstorage.service.networklayer.com" + endpoint>1 +``` + + +7. Specify a IBM COS Location Constraint. The location constraint must match endpoint when using IBM Cloud Public. For on-prem COS, do not make a selection from this list, hit enter +``` + 1 / US Cross Region Standard + \ "us-standard" + 2 / US Cross Region Vault + \ "us-vault" + 3 / US Cross Region Cold + \ "us-cold" + 4 / US Cross Region Flex + \ "us-flex" + 5 / US East Region Standard + \ "us-east-standard" + 6 / US East Region Vault + \ "us-east-vault" + 7 / US East Region Cold + \ "us-east-cold" + 8 / US East Region Flex + \ "us-east-flex" + 9 / US South Region Standard + \ "us-south-standard" + 10 / US South Region Vault + \ "us-south-vault" +[snip] + 32 / Toronto Flex + \ "tor01-flex" +location_constraint>1 +``` + +9. Specify a canned ACL. IBM Cloud (Strorage) supports "public-read" and "private". IBM Cloud(Infra) supports all the canned ACLs. On-Premise COS supports all the canned ACLs. +``` +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise COS + \ "private" + 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access. This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise IBM COS + \ "public-read" + 3 / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. This acl is available on IBM Cloud (Infra), On-Premise IBM COS + \ "public-read-write" + 4 / Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS + \ "authenticated-read" +acl> 1 +``` + + +12. Review the displayed configuration and accept to save the "remote" then quit. The config file should look like this +``` + [xxx] + type = s3 + Provider = IBMCOS + access_key_id = xxx + secret_access_key = yyy + endpoint = s3-api.us-geo.objectstorage.softlayer.net + location_constraint = us-standard + acl = private +``` + +13. Execute rclone commands +``` + 1) Create a bucket. + rclone mkdir IBM-COS-XREGION:newbucket + 2) List available buckets. + rclone lsd IBM-COS-XREGION: + -1 2017-11-08 21:16:22 -1 test + -1 2018-02-14 20:16:39 -1 newbucket + 3) List contents of a bucket. + rclone ls IBM-COS-XREGION:newbucket + 18685952 test.exe + 4) Copy a file from local to remote. + rclone copy /Users/file.txt IBM-COS-XREGION:newbucket + 5) Copy a file from remote to local. + rclone copy IBM-COS-XREGION:newbucket/file.txt . + 6) Delete a file on remote. + rclone delete IBM-COS-XREGION:newbucket/file.txt +``` + +### Minio ### + +[Minio](https://minio.io/) is an object storage server built for cloud application developers and devops. + +It is very easy to install and provides an S3 compatible server which can be used by rclone. + +To use it, install Minio following the instructions [here](https://docs.minio.io/docs/minio-quickstart-guide). + +When it configures itself Minio will print something like this + +``` +Endpoint: http://192.168.1.106:9000 http://172.23.0.1:9000 +AccessKey: USWUXHGYZQYFYFFIT3RE +SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 +Region: us-east-1 +SQS ARNs: arn:minio:sqs:us-east-1:1:redis arn:minio:sqs:us-east-1:2:redis + +Browser Access: + http://192.168.1.106:9000 http://172.23.0.1:9000 + +Command-line Access: https://docs.minio.io/docs/minio-client-quickstart-guide + $ mc config host add myminio http://192.168.1.106:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 + +Object API (Amazon S3 compatible): + Go: https://docs.minio.io/docs/golang-client-quickstart-guide + Java: https://docs.minio.io/docs/java-client-quickstart-guide + Python: https://docs.minio.io/docs/python-client-quickstart-guide + JavaScript: https://docs.minio.io/docs/javascript-client-quickstart-guide + .NET: https://docs.minio.io/docs/dotnet-client-quickstart-guide + +Drive Capacity: 26 GiB Free, 165 GiB Total +``` + +These details need to go into `rclone config` like this. Note that it +is important to put the region in as stated above. + +``` +env_auth> 1 +access_key_id> USWUXHGYZQYFYFFIT3RE +secret_access_key> MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 +region> us-east-1 +endpoint> http://192.168.1.106:9000 +location_constraint> +server_side_encryption> +``` + +Which makes the config file look like this + +``` +[minio] +type = s3 +provider = Minio +env_auth = false +access_key_id = USWUXHGYZQYFYFFIT3RE +secret_access_key = MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03 +region = us-east-1 +endpoint = http://192.168.1.106:9000 +location_constraint = +server_side_encryption = +``` + +So once set up, for example to copy files into a bucket + +``` +rclone copy /path/to/files minio:bucket +``` + +### Wasabi ### + +[Wasabi](https://wasabi.com) is a cloud-based object storage service for a +broad range of applications and use cases. Wasabi is designed for +individuals and organizations that require a high-performance, +reliable, and secure data storage infrastructure at minimal cost. + +Wasabi provides an S3 interface which can be configured for use with +rclone like this. + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +n/s> n +name> wasabi +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" +[snip] +Storage> s3 +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" +env_auth> 1 +AWS Access Key ID - leave blank for anonymous access or runtime credentials. +access_key_id> YOURACCESSKEY +AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. +secret_access_key> YOURSECRETACCESSKEY +Region to connect to. +Choose a number from below, or type in your own value + / The default endpoint - a good choice if you are unsure. + 1 | US Region, Northern Virginia or Pacific Northwest. + | Leave location constraint empty. + \ "us-east-1" +[snip] +region> us-east-1 +Endpoint for S3 API. +Leave blank if using AWS to use the default endpoint for the region. +Specify if using an S3 clone such as Ceph. +endpoint> s3.wasabisys.com +Location constraint - must be set to match the Region. Used when creating buckets only. +Choose a number from below, or type in your own value + 1 / Empty for US Region, Northern Virginia or Pacific Northwest. + \ "" +[snip] +location_constraint> +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" +[snip] +acl> +The server-side encryption algorithm used when storing this object in S3. +Choose a number from below, or type in your own value + 1 / None + \ "" + 2 / AES256 + \ "AES256" +server_side_encryption> +The storage class to use when storing objects in S3. +Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Standard storage class + \ "STANDARD" + 3 / Reduced redundancy storage class + \ "REDUCED_REDUNDANCY" + 4 / Standard Infrequent Access storage class + \ "STANDARD_IA" +storage_class> +Remote config +-------------------- +[wasabi] +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +region = us-east-1 +endpoint = s3.wasabisys.com +location_constraint = +acl = +server_side_encryption = +storage_class = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This will leave the config file looking like this. + +``` +[wasabi] +type = s3 +provider = Wasabi +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +region = +endpoint = s3.wasabisys.com +location_constraint = +acl = +server_side_encryption = +storage_class = +``` + +### Aliyun OSS / Netease NOS ### + +This describes how to set up Aliyun OSS - Netease NOS is the same +except for different endpoints. + +Note this is a pretty standard S3 setup, except for the setting of +`force_path_style = false` in the advanced config. + +``` +# rclone config +e/n/d/r/c/s/q> n +name> oss +Type of storage to configure. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio) + \ "s3" +Storage> s3 +Choose your S3 provider. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 8 / Any other S3 compatible provider + \ "Other" +provider> other +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Enter a boolean value (true or false). Press Enter for the default ("false"). +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" +env_auth> 1 +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a string value. Press Enter for the default (""). +access_key_id> xxxxxxxxxxxx +AWS Secret Access Key (password) +Leave blank for anonymous access or runtime credentials. +Enter a string value. Press Enter for the default (""). +secret_access_key> xxxxxxxxxxxxxxxxx +Region to connect to. +Leave blank if you are using an S3 clone and you don't have a region. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 1 / Use this if unsure. Will use v4 signatures and an empty region. + \ "" + 2 / Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH. + \ "other-v2-signature" +region> 1 +Endpoint for S3 API. +Required when using an S3 clone. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value +endpoint> oss-cn-shenzhen.aliyuncs.com +Location constraint - must be set to match the Region. +Leave blank if not sure. Used when creating buckets only. +Enter a string value. Press Enter for the default (""). +location_constraint> +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" +acl> 1 +Edit advanced config? (y/n) +y) Yes +n) No +y/n> y +Chunk size to use for uploading +Enter a size with suffix k,M,G,T. Press Enter for the default ("5M"). +chunk_size> +Don't store MD5 checksum with object metadata +Enter a boolean value (true or false). Press Enter for the default ("false"). +disable_checksum> +An AWS session token +Enter a string value. Press Enter for the default (""). +session_token> +Concurrency for multipart uploads. +Enter a signed integer. Press Enter for the default ("2"). +upload_concurrency> +If true use path style access if false use virtual hosted style. +Some providers (eg Aliyun OSS or Netease COS) require this. +Enter a boolean value (true or false). Press Enter for the default ("true"). +force_path_style> false +Remote config +-------------------- +[oss] +type = s3 +provider = Other +env_auth = false +access_key_id = xxxxxxxxx +secret_access_key = xxxxxxxxxxxxx +endpoint = oss-cn-shenzhen.aliyuncs.com +acl = private +force_path_style = false +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` diff --git a/.rclone_repo/docs/content/sftp.md b/.rclone_repo/docs/content/sftp.md new file mode 100755 index 0000000..8110631 --- /dev/null +++ b/.rclone_repo/docs/content/sftp.md @@ -0,0 +1,212 @@ +--- +title: "SFTP" +description: "SFTP" +date: "2017-02-01" +--- + + SFTP +---------------------------------------- + +SFTP is the [Secure (or SSH) File Transfer +Protocol](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol). + +SFTP runs over SSH v2 and is installed as standard with most modern +SSH installations. + +Paths are specified as `remote:path`. If the path does not begin with +a `/` it is relative to the home directory of the user. An empty path +`remote:` refers to the user's home directory. + +Note that some SFTP servers will need the leading `/` - Synology is a +good example of this. + +Here is an example of making an SFTP configuration. First run + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / FTP Connection + \ "ftp" + 7 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 8 / Google Drive + \ "drive" + 9 / Hubic + \ "hubic" +10 / Local Disk + \ "local" +11 / Microsoft OneDrive + \ "onedrive" +12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +13 / SSH/SFTP Connection + \ "sftp" +14 / Yandex Disk + \ "yandex" +15 / http Connection + \ "http" +Storage> sftp +SSH host to connect to +Choose a number from below, or type in your own value + 1 / Connect to example.com + \ "example.com" +host> example.com +SSH username, leave blank for current username, ncw +user> sftpuser +SSH port, leave blank to use default (22) +port> +SSH password, leave blank to use ssh-agent. +y) Yes type in my own password +g) Generate random password +n) No leave this optional password blank +y/g/n> n +Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent. +key_file> +Remote config +-------------------- +[remote] +host = example.com +user = sftpuser +port = +pass = +key_file = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This remote is called `remote` and can now be used like this: + +See all directories in the home directory + + rclone lsd remote: + +Make a new directory + + rclone mkdir remote:path/to/directory + +List the contents of a directory + + rclone ls remote:path/to/directory + +Sync `/home/local/directory` to the remote directory, deleting any +excess files in the directory. + + rclone sync /home/local/directory remote:directory + +### SSH Authentication ### + +The SFTP remote supports three authentication methods: + + * Password + * Key file + * ssh-agent + +Key files should be unencrypted PEM-encoded private key files. For +instance `/home/$USER/.ssh/id_rsa`. + +If you don't specify `pass` or `key_file` then rclone will attempt to +contact an ssh-agent. + +If you set the `--sftp-ask-password` option, rclone will prompt for a +password when needed and no password has been configured. + +### ssh-agent on macOS ### + +Note that there seem to be various problems with using an ssh-agent on +macOS due to recent changes in the OS. The most effective work-around +seems to be to start an ssh-agent in each session, eg + + eval `ssh-agent -s` && ssh-add -A + +And then at the end of the session + + eval `ssh-agent -k` + +These commands can be used in scripts of course. + +### Specific options ### + +Here are the command line options specific to this remote. + +#### --sftp-ask-password #### + +Ask for the SFTP password if needed when no password has been configured. + +#### --ssh-path-override #### + +Override path used by SSH connection. Allows checksum calculation when +SFTP and SSH paths are different. This issue affects among others Synology +NAS boxes. + +Shared folders can be found in directories representing volumes + + rclone sync /home/local/directory remote:/directory --ssh-path-override /volume2/directory + +Home directory can be found in a shared folder called `homes` + + rclone sync /home/local/directory remote:/home/directory --ssh-path-override /volume1/homes/USER/directory + +### Modified time ### + +Modified times are stored on the server to 1 second precision. + +Modified times are used in syncing and are fully supported. + +Some SFTP servers disable setting/modifying the file modification time after +upload (for example, certain configurations of ProFTPd with mod_sftp). If you +are using one of these servers, you can set the option `set_modtime = false` in +your RClone backend configuration to disable this behaviour. + +### Limitations ### + +SFTP supports checksums if the same login has shell access and `md5sum` +or `sha1sum` as well as `echo` are in the remote's PATH. +This remote checksumming (file hashing) is recommended and enabled by default. +Disabling the checksumming may be required if you are connecting to SFTP servers +which are not under your control, and to which the execution of remote commands +is prohibited. Set the configuration option `disable_hashcheck` to `true` to +disable checksumming. + +Note that some SFTP servers (eg Synology) the paths are different for +SSH and SFTP so the hashes can't be calculated properly. For them +using `disable_hashcheck` is a good idea. + +The only ssh agent supported under Windows is Putty's pageant. + +The Go SSH library disables the use of the aes128-cbc cipher by +default, due to security concerns. This can be re-enabled on a +per-connection basis by setting the `use_insecure_cipher` setting in +the configuration file to `true`. Further details on the insecurity of +this cipher can be found [in this paper] +(http://www.isg.rhul.ac.uk/~kp/SandPfinal.pdf). + +SFTP isn't supported under plan9 until [this +issue](https://github.com/pkg/sftp/issues/156) is fixed. + +Note that since SFTP isn't HTTP based the following flags don't work +with it: `--dump-headers`, `--dump-bodies`, `--dump-auth` + +Note that `--timeout` isn't supported (but `--contimeout` is). diff --git a/.rclone_repo/docs/content/swift.md b/.rclone_repo/docs/content/swift.md new file mode 100755 index 0000000..6152245 --- /dev/null +++ b/.rclone_repo/docs/content/swift.md @@ -0,0 +1,311 @@ +--- +title: "Swift" +description: "Swift" +date: "2014-04-26" +--- + +Swift +---------------------------------------- + +Swift refers to [Openstack Object Storage](https://docs.openstack.org/swift/latest/). +Commercial implementations of that being: + + * [Rackspace Cloud Files](https://www.rackspace.com/cloud/files/) + * [Memset Memstore](https://www.memset.com/cloud/storage/) + * [OVH Object Storage](https://www.ovh.co.uk/public-cloud/storage/object-storage/) + * [Oracle Cloud Storage](https://cloud.oracle.com/storage-opc) + * [IBM Bluemix Cloud ObjectStorage Swift](https://console.bluemix.net/docs/infrastructure/objectstorage-swift/index.html) + +Paths are specified as `remote:container` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, eg `remote:container/path/to/dir`. + +Here is an example of making a swift configuration. First run + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Box + \ "box" + 5 / Cache a remote + \ "cache" + 6 / Dropbox + \ "dropbox" + 7 / Encrypt/Decrypt a remote + \ "crypt" + 8 / FTP Connection + \ "ftp" + 9 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" +10 / Google Drive + \ "drive" +11 / Hubic + \ "hubic" +12 / Local Disk + \ "local" +13 / Microsoft Azure Blob Storage + \ "azureblob" +14 / Microsoft OneDrive + \ "onedrive" +15 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +16 / Pcloud + \ "pcloud" +17 / QingCloud Object Storage + \ "qingstor" +18 / SSH/SFTP Connection + \ "sftp" +19 / Webdav + \ "webdav" +20 / Yandex Disk + \ "yandex" +21 / http Connection + \ "http" +Storage> swift +Get swift credentials from environment variables in standard OpenStack form. +Choose a number from below, or type in your own value + 1 / Enter swift credentials in the next step + \ "false" + 2 / Get swift credentials from environment vars. Leave other fields blank if using this. + \ "true" +env_auth> true +User name to log in (OS_USERNAME). +user> +API key or password (OS_PASSWORD). +key> +Authentication URL for server (OS_AUTH_URL). +Choose a number from below, or type in your own value + 1 / Rackspace US + \ "https://auth.api.rackspacecloud.com/v1.0" + 2 / Rackspace UK + \ "https://lon.auth.api.rackspacecloud.com/v1.0" + 3 / Rackspace v2 + \ "https://identity.api.rackspacecloud.com/v2.0" + 4 / Memset Memstore UK + \ "https://auth.storage.memset.com/v1.0" + 5 / Memset Memstore UK v2 + \ "https://auth.storage.memset.com/v2.0" + 6 / OVH + \ "https://auth.cloud.ovh.net/v2.0" +auth> +User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID). +user_id> +User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME) +domain> +Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME) +tenant> +Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID) +tenant_id> +Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME) +tenant_domain> +Region name - optional (OS_REGION_NAME) +region> +Storage URL - optional (OS_STORAGE_URL) +storage_url> +Auth Token from alternate authentication - optional (OS_AUTH_TOKEN) +auth_token> +AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION) +auth_version> +Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) +Choose a number from below, or type in your own value + 1 / Public (default, choose this if not sure) + \ "public" + 2 / Internal (use internal service net) + \ "internal" + 3 / Admin + \ "admin" +endpoint_type> +Remote config +-------------------- +[test] +env_auth = true +user = +key = +auth = +user_id = +domain = +tenant = +tenant_id = +tenant_domain = +region = +storage_url = +auth_token = +auth_version = +endpoint_type = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This remote is called `remote` and can now be used like this + +See all containers + + rclone lsd remote: + +Make a new container + + rclone mkdir remote:container + +List the contents of a container + + rclone ls remote:container + +Sync `/home/local/directory` to the remote container, deleting any +excess files in the container. + + rclone sync /home/local/directory remote:container + +### Configuration from an Openstack credentials file ### + +An Opentstack credentials file typically looks something something +like this (without the comments) + +``` +export OS_AUTH_URL=https://a.provider.net/v2.0 +export OS_TENANT_ID=ffffffffffffffffffffffffffffffff +export OS_TENANT_NAME="1234567890123456" +export OS_USERNAME="123abc567xy" +echo "Please enter your OpenStack Password: " +read -sr OS_PASSWORD_INPUT +export OS_PASSWORD=$OS_PASSWORD_INPUT +export OS_REGION_NAME="SBG1" +if [ -z "$OS_REGION_NAME" ]; then unset OS_REGION_NAME; fi +``` + +The config file needs to look something like this where `$OS_USERNAME` +represents the value of the `OS_USERNAME` variable - `123abc567xy` in +the example above. + +``` +[remote] +type = swift +user = $OS_USERNAME +key = $OS_PASSWORD +auth = $OS_AUTH_URL +tenant = $OS_TENANT_NAME +``` + +Note that you may (or may not) need to set `region` too - try without first. + +### Configuration from the environment ### + +If you prefer you can configure rclone to use swift using a standard +set of OpenStack environment variables. + +When you run through the config, make sure you choose `true` for +`env_auth` and leave everything else blank. + +rclone will then set any empty config parameters from the environment +using standard OpenStack environment variables. There is [a list of +the +variables](https://godoc.org/github.com/ncw/swift#Connection.ApplyEnvironment) +in the docs for the swift library. + +### Using an alternate authentication method ### + +If your OpenStack installation uses a non-standard authentication method +that might not be yet supported by rclone or the underlying swift library, +you can authenticate externally (e.g. calling manually the `openstack` +commands to get a token). Then, you just need to pass the two +configuration variables ``auth_token`` and ``storage_url``. +If they are both provided, the other variables are ignored. rclone will +not try to authenticate but instead assume it is already authenticated +and use these two variables to access the OpenStack installation. + +#### Using rclone without a config file #### + +You can use rclone with swift without a config file, if desired, like +this: + +``` +source openstack-credentials-file +export RCLONE_CONFIG_MYREMOTE_TYPE=swift +export RCLONE_CONFIG_MYREMOTE_ENV_AUTH=true +rclone lsd myremote: +``` + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +### --update and --use-server-modtime ### + +As noted below, the modified time is stored on metadata on the object. It is +used by default for all operations that require checking the time a file was +last updated. It allows rclone to treat the remote more like a true filesystem, +but it is inefficient because it requires an extra API call to retrieve the +metadata. + +For many operations, the time the object was last uploaded to the remote is +sufficient to determine if it is "dirty". By using `--update` along with +`--use-server-modtime`, you can avoid the extra API call and simply upload +files whose local modtime is newer than the time it was last uploaded. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --swift-storage-policy=STRING #### +Apply the specified storage policy when creating a new container. The policy +cannot be changed afterwards. The allowed configuration values and their +meaning depend on your Swift storage provider. + +#### --swift-chunk-size=SIZE #### + +Above this size files will be chunked into a _segments container. The +default for this is 5GB which is its maximum value. + +### Modified time ### + +The modified time is stored as metadata on the object as +`X-Object-Meta-Mtime` as floating point since the epoch accurate to 1 +ns. + +This is a defacto standard (used in the official python-swiftclient +amongst others) for storing the modification time for an object. + +### Limitations ### + +The Swift API doesn't return a correct MD5SUM for segmented files +(Dynamic or Static Large Objects) so rclone won't check or use the +MD5SUM for these. + +### Troubleshooting ### + +#### Rclone gives Failed to create file system for "remote:": Bad Request #### + +Due to an oddity of the underlying swift library, it gives a "Bad +Request" error rather than a more sensible error when the +authentication fails for Swift. + +So this most likely means your username / password is wrong. You can +investigate further with the `--dump-bodies` flag. + +This may also be caused by specifying the region when you shouldn't +have (eg OVH). + +#### Rclone gives Failed to create file system: Response didn't have storage storage url and auth token #### + +This is most likely caused by forgetting to specify your tenant when +setting up a swift remote. diff --git a/.rclone_repo/docs/content/webdav.md b/.rclone_repo/docs/content/webdav.md new file mode 100755 index 0000000..c85a99e --- /dev/null +++ b/.rclone_repo/docs/content/webdav.md @@ -0,0 +1,211 @@ +--- +title: "WebDAV" +description: "Rclone docs for WebDAV" +date: "2017-10-01" +--- + + WebDAV +----------------------------------------- + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +To configure the WebDAV remote you will need to have a URL for it, and +a username and password. If you know what kind of system you are +connecting to then rclone can enable extra features. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value +[snip] +22 / Webdav + \ "webdav" +[snip] +Storage> webdav +URL of http host to connect to +Choose a number from below, or type in your own value + 1 / Connect to example.com + \ "https://example.com" +url> https://example.com/remote.php/webdav/ +Name of the Webdav site/service/software you are using +Choose a number from below, or type in your own value + 1 / Nextcloud + \ "nextcloud" + 2 / Owncloud + \ "owncloud" + 3 / Sharepoint + \ "sharepoint" + 4 / Other site/service or software + \ "other" +vendor> 1 +User name +user> user +Password. +y) Yes type in my own password +g) Generate random password +n) No leave this optional password blank +y/g/n> y +Enter the password: +password: +Confirm the password: +password: +Bearer token instead of user/pass (eg a Macaroon) +bearer_token> +Remote config +-------------------- +[remote] +type = webdav +url = https://example.com/remote.php/webdav/ +vendor = nextcloud +user = user +pass = *** ENCRYPTED *** +bearer_token = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Once configured you can then use `rclone` like this, + +List directories in top level of your WebDAV + + rclone lsd remote: + +List all the files in your WebDAV + + rclone ls remote: + +To copy a local directory to an WebDAV directory called backup + + rclone copy /home/source remote:backup + +### Modified time and hashes ### + +Plain WebDAV does not support modified times. However when used with +Owncloud or Nextcloud rclone will support modified times. + +Hashes are not supported. + +## Provider notes ## + +See below for notes on specific providers. + +### Owncloud ### + +Click on the settings cog in the bottom right of the page and this +will show the WebDAV URL that rclone needs in the config step. It +will look something like `https://example.com/remote.php/webdav/`. + +Owncloud supports modified times using the `X-OC-Mtime` header. + +### Nextcloud ### + +This is configured in an identical way to Owncloud. Note that +Nextcloud does not support streaming of files (`rcat`) whereas +Owncloud does. This [may be +fixed](https://github.com/nextcloud/nextcloud-snap/issues/365) in the +future. + +### Put.io ### + +put.io can be accessed in a read only way using webdav. + +Configure the `url` as `https://webdav.put.io` and use your normal +account username and password for `user` and `pass`. Set the `vendor` +to `other`. + +Your config file should end up looking like this: + +``` +[putio] +type = webdav +url = https://webdav.put.io +vendor = other +user = YourUserName +pass = encryptedpassword +``` + +If you are using `put.io` with `rclone mount` then use the +`--read-only` flag to signal to the OS that it can't write to the +mount. + +For more help see [the put.io webdav docs](http://help.put.io/apps-and-integrations/ftp-and-webdav). + +### Sharepoint ### + +Rclone can be used with Sharepoint provided by OneDrive for Business +or Office365 Education Accounts. +This feature is only needed for a few of these Accounts, +mostly Office365 Education ones. These accounts are sometimes not +verified by the domain owner [github#1975](https://github.com/ncw/rclone/issues/1975) + +This means that these accounts can't be added using the official +API (other Accounts should work with the "onedrive" option). However, +it is possible to access them using webdav. + +To use a sharepoint remote with rclone, add it like this: +First, you need to get your remote's URL: + +- Go [here](https://onedrive.live.com/about/en-us/signin/) + to open your OneDrive or to sign in +- Now take a look at your address bar, the URL should look like this: + `https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/_layouts/15/onedrive.aspx` + +You'll only need this URL upto the email address. After that, you'll +most likely want to add "/Documents". That subdirectory contains +the actual data stored on your OneDrive. + +Add the remote to rclone like this: +Configure the `url` as `https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/Documents` +and use your normal account email and password for `user` and `pass`. +If you have 2FA enabled, you have to generate an app password. +Set the `vendor` to `sharepoint`. + +Your config file should look like this: + +``` +[sharepoint] +type = webdav +url = https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/Documents +vendor = other +user = YourEmailAddress +pass = encryptedpassword +``` + +### dCache ### + +dCache is a storage system with WebDAV doors that support, beside basic and x509, +authentication with [Macaroons](https://www.dcache.org/manuals/workshop-2017-05-29-Umea/000-Final/anupam_macaroons_v02.pdf) (bearer tokens). + +Configure as normal using the `other` type. Don't enter a username or +password, instead enter your Macaroon as the `bearer_token`. + +The config will end up looking something like this. + +``` +[dcache] +type = webdav +url = https://dcache... +vendor = other +user = +pass = +bearer_token = your-macaroon +``` + +There is a [script](https://github.com/onnozweers/dcache-scripts/blob/master/get-share-link) that +obtains a Macaroon from a dCache WebDAV endpoint, and creates an rclone config file. diff --git a/.rclone_repo/docs/content/yandex.md b/.rclone_repo/docs/content/yandex.md new file mode 100755 index 0000000..91bff66 --- /dev/null +++ b/.rclone_repo/docs/content/yandex.md @@ -0,0 +1,129 @@ +--- +title: "Yandex" +description: "Yandex Disk" +date: "2015-12-30" +--- + +Yandex Disk +---------------------------------------- + +[Yandex Disk](https://disk.yandex.com) is a cloud storage solution created by [Yandex](https://yandex.com). + +Yandex paths may be as deep as required, eg `remote:directory/subdirectory`. + +Here is an example of making a yandex configuration. First run + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +n/s> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" +10 / Microsoft OneDrive + \ "onedrive" +11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +12 / SSH/SFTP Connection + \ "sftp" +13 / Yandex Disk + \ "yandex" +Storage> 13 +Yandex Client Id - leave blank normally. +client_id> +Yandex Client Secret - leave blank normally. +client_secret> +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +client_id = +client_secret = +token = {"access_token":"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","token_type":"bearer","expiry":"2016-12-29T12:27:11.362788025Z"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +See the [remote setup docs](/remote_setup/) for how to set it up on a +machine with no Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Yandex Disk. This only runs from the moment it +opens your browser to the moment you get back the verification code. +This is on `http://127.0.0.1:53682/` and this it may require you to +unblock it temporarily if you are running a host firewall. + +Once configured you can then use `rclone` like this, + +See top level directories + + rclone lsd remote: + +Make a new directory + + rclone mkdir remote:directory + +List the contents of a directory + + rclone ls remote:directory + +Sync `/home/local/directory` to the remote path, deleting any +excess files in the path. + + rclone sync /home/local/directory remote:directory + +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + +### Modified time ### + +Modified times are supported and are stored accurate to 1 ns in custom +metadata called `rclone_modified` in RFC3339 with nanoseconds format. + +### MD5 checksums ### + +MD5 checksums are natively supported by Yandex Disk. + +### Emptying Trash ### + +If you wish to empty your trash you can use the `rclone cleanup remote:` +command which will permanently delete all your trashed files. This command +does not take any path arguments. diff --git a/.rclone_repo/docs/i18n/en.toml b/.rclone_repo/docs/i18n/en.toml new file mode 100755 index 0000000..b5910c6 --- /dev/null +++ b/.rclone_repo/docs/i18n/en.toml @@ -0,0 +1,5 @@ +# Dummy translation file to make errors go away +# See: https://discourse.gohugo.io/t/monolingual-site/6300 + +[wordCount] +other = "{{ .WordCount }} words" diff --git a/.rclone_repo/docs/layouts/404.html b/.rclone_repo/docs/layouts/404.html new file mode 100755 index 0000000..65387aa --- /dev/null +++ b/.rclone_repo/docs/layouts/404.html @@ -0,0 +1,7 @@ +{{ define "main"}} +
+
+

Go Home

+
+
+{{ end }} diff --git a/.rclone_repo/docs/layouts/_default/single.html b/.rclone_repo/docs/layouts/_default/single.html new file mode 100755 index 0000000..e81ddda --- /dev/null +++ b/.rclone_repo/docs/layouts/_default/single.html @@ -0,0 +1,21 @@ +{{ template "chrome/header.html" . }} + +{{ template "chrome/navbar.html" . }} +
+
+
+
+

{{ .Title }}
{{ .Description }}

+
+ {{ .Content }} +
+
+ + +
+ {{ template "chrome/menu.html" . }} +
+
+{{ template "chrome/footer.copyright.html" . }} +
+{{ template "chrome/footer.html" . }} diff --git a/.rclone_repo/docs/layouts/chrome/footer.copyright.html b/.rclone_repo/docs/layouts/chrome/footer.copyright.html new file mode 100755 index 0000000..94b8b9f --- /dev/null +++ b/.rclone_repo/docs/layouts/chrome/footer.copyright.html @@ -0,0 +1,11 @@ + diff --git a/.rclone_repo/docs/layouts/chrome/footer.html b/.rclone_repo/docs/layouts/chrome/footer.html new file mode 100755 index 0000000..4ec8c6d --- /dev/null +++ b/.rclone_repo/docs/layouts/chrome/footer.html @@ -0,0 +1,6 @@ + + + + + + diff --git a/.rclone_repo/docs/layouts/chrome/header.html b/.rclone_repo/docs/layouts/chrome/header.html new file mode 100755 index 0000000..d9bfa10 --- /dev/null +++ b/.rclone_repo/docs/layouts/chrome/header.html @@ -0,0 +1,9 @@ + + + + {{ template "chrome/meta.html" . }} + {{ .Title }} + + {{ template "chrome/header.includes.html" . }} + {{ if .RSSLink }}{{ end }} + diff --git a/.rclone_repo/docs/layouts/chrome/header.includes.html b/.rclone_repo/docs/layouts/chrome/header.includes.html new file mode 100755 index 0000000..3d3a370 --- /dev/null +++ b/.rclone_repo/docs/layouts/chrome/header.includes.html @@ -0,0 +1,3 @@ + + + diff --git a/.rclone_repo/docs/layouts/chrome/menu.html b/.rclone_repo/docs/layouts/chrome/menu.html new file mode 100755 index 0000000..f16e452 --- /dev/null +++ b/.rclone_repo/docs/layouts/chrome/menu.html @@ -0,0 +1,30 @@ +
+
+

Share and Enjoy.

+
+
+ + +
+ +
+
+
+
+ +
+
+

Links.

+
+
+

+ Rclone forum.
+ Github project.
+ Google+ page.
+ Rclone slack chat.
+ Rclone Wiki.
+ Donate.
+ @njcw +

+
+
diff --git a/.rclone_repo/docs/layouts/chrome/meta.html b/.rclone_repo/docs/layouts/chrome/meta.html new file mode 100755 index 0000000..ff5b856 --- /dev/null +++ b/.rclone_repo/docs/layouts/chrome/meta.html @@ -0,0 +1,14 @@ + + + + + + + diff --git a/.rclone_repo/docs/layouts/chrome/navbar.html b/.rclone_repo/docs/layouts/chrome/navbar.html new file mode 100755 index 0000000..f783776 --- /dev/null +++ b/.rclone_repo/docs/layouts/chrome/navbar.html @@ -0,0 +1,87 @@ + diff --git a/.rclone_repo/docs/layouts/index.html b/.rclone_repo/docs/layouts/index.html new file mode 100755 index 0000000..264e1de --- /dev/null +++ b/.rclone_repo/docs/layouts/index.html @@ -0,0 +1,19 @@ +{{ template "chrome/header.html" . }} + +{{ template "chrome/navbar.html" . }} +
+
+
+ {{ range $key, $value := .Site.Taxonomies.groups.about.Pages }} + {{ $value.Content }} + {{ end }} +
+ + +
+ {{ template "chrome/menu.html" . }} +
+
+{{ template "chrome/footer.copyright.html" . }} +
+{{ template "chrome/footer.html" . }} diff --git a/.rclone_repo/docs/layouts/indexes/category.html b/.rclone_repo/docs/layouts/indexes/category.html new file mode 100755 index 0000000..0655674 --- /dev/null +++ b/.rclone_repo/docs/layouts/indexes/category.html @@ -0,0 +1,24 @@ +{{ template "chrome/header.html" . }} + +{{ template "chrome/navbar.html" . }} +
+
+
+
+ Items in category {{ .Title | lower }} +
    + {{ range .Data.Pages }} + {{ .Render "li" }} + {{ end}} +
+
+
+ + +
+ {{ template "chrome/menu.html" . }} +
+
+{{ template "chrome/footer.copyright.html" . }} +
+{{ template "chrome/footer.html" . }} diff --git a/.rclone_repo/docs/layouts/indexes/post.html b/.rclone_repo/docs/layouts/indexes/post.html new file mode 100755 index 0000000..85ee16c --- /dev/null +++ b/.rclone_repo/docs/layouts/indexes/post.html @@ -0,0 +1,24 @@ +{{ template "chrome/header.html" . }} + +{{ template "chrome/navbar.html" . }} +
+
+
+
+ Blog Post Archive +
    + {{ range .Data.Pages }} + {{ .Render "li" }} + {{ end}} +
+
+
+ + +
+ {{ template "chrome/menu.html" . }} +
+
+{{ template "chrome/footer.copyright.html" . }} +
+{{ template "chrome/footer.html" . }} diff --git a/.rclone_repo/docs/layouts/indexes/tag.html b/.rclone_repo/docs/layouts/indexes/tag.html new file mode 100755 index 0000000..e862f26 --- /dev/null +++ b/.rclone_repo/docs/layouts/indexes/tag.html @@ -0,0 +1,24 @@ +{{ template "chrome/header.html" . }} + +{{ template "chrome/navbar.html" . }} +
+
+
+
+ Items with tag {{ .Title | lower }} +
    + {{ range .Data.Pages }} + {{ .Render "li" }} + {{ end}} +
+
+
+ + +
+ {{ template "chrome/menu.html" . }} +
+
+{{ template "chrome/footer.copyright.html" . }} +
+{{ template "chrome/footer.html" . }} diff --git a/.rclone_repo/docs/layouts/page/single.html b/.rclone_repo/docs/layouts/page/single.html new file mode 100755 index 0000000..589fa8d --- /dev/null +++ b/.rclone_repo/docs/layouts/page/single.html @@ -0,0 +1,17 @@ +{{ template "chrome/header.html" . }} + +{{ template "chrome/navbar.html" . }} +
+
+
+ {{ .Content }} +
+ + +
+ {{ template "chrome/menu.html" . }} +
+
+ {{ template "chrome/footer.copyright.html" . }} +
+{{ template "chrome/footer.html" . }} diff --git a/.rclone_repo/docs/layouts/partials/version.html b/.rclone_repo/docs/layouts/partials/version.html new file mode 100755 index 0000000..503ae8b --- /dev/null +++ b/.rclone_repo/docs/layouts/partials/version.html @@ -0,0 +1 @@ +v1.43.1 \ No newline at end of file diff --git a/.rclone_repo/docs/layouts/post/li.html b/.rclone_repo/docs/layouts/post/li.html new file mode 100755 index 0000000..d57be9c --- /dev/null +++ b/.rclone_repo/docs/layouts/post/li.html @@ -0,0 +1,4 @@ +
  • +
    {{ .Title}}
    + posted on {{ .Date.Format "January 2, 2006" }}
    +
    • \ No newline at end of file diff --git a/.rclone_repo/docs/layouts/post/single.html b/.rclone_repo/docs/layouts/post/single.html new file mode 100755 index 0000000..f32cac9 --- /dev/null +++ b/.rclone_repo/docs/layouts/post/single.html @@ -0,0 +1,35 @@ +{{ template "chrome/header.html" . }} + +{{ template "chrome/navbar.html" . }} +
    +
    +
    +
    +

    {{ .Title }}
    {{ .Description }}

    +
    + {{ .Content }} +
    +
    + + +
    +
    +

    {{ .Date.Format "January 2, 2006" }}
    + {{ .WordCount }} words

    +
    + Categories +
      + {{ range .Params.categories }} +
    • {{ . }}
      • + {{ end }} +
    +
    + Tags
    + {{ range .Params.tags }}{{ . }} {{ end }} +
    + {{ template "chrome/menu.html" . }} +
    +
    +{{ template "chrome/footer.copyright.html" . }} +
    +{{ template "chrome/footer.html" . }} diff --git a/.rclone_repo/docs/layouts/post/summary.html b/.rclone_repo/docs/layouts/post/summary.html new file mode 100755 index 0000000..f70b682 --- /dev/null +++ b/.rclone_repo/docs/layouts/post/summary.html @@ -0,0 +1,9 @@ +
    +

    + {{ .Title }} Posted on {{ .Date.Format "Jan 2, 2006" }}
    + {{ .Description }} +

    +
    +

    {{ .Summary }}

    + Read More +
    \ No newline at end of file diff --git a/.rclone_repo/docs/layouts/rss.xml b/.rclone_repo/docs/layouts/rss.xml new file mode 100755 index 0000000..43960a2 --- /dev/null +++ b/.rclone_repo/docs/layouts/rss.xml @@ -0,0 +1,20 @@ + + + {{ .Title }} on {{ .Site.Title }} + {{ .Permalink }} + en-US + Nick Craig-Wood + Copyright (c) 2017, Nick Craig-Wood; all rights reserved. + {{ .Date.Format "Mon, 02 Jan 2006 15:04:05 MST" }} + {{ range first 15 .Data.Pages }} + + {{ .Title }} + {{ .Permalink }} + {{ .Date.Format "Mon, 02 Jan 2006 15:04:05 MST" }} + Nick Craig-Wood + {{ .Permalink }} + {{ .Content | html }} + + {{ end }} + + diff --git a/.rclone_repo/docs/layouts/section/commands.html b/.rclone_repo/docs/layouts/section/commands.html new file mode 100755 index 0000000..ad6f9e3 --- /dev/null +++ b/.rclone_repo/docs/layouts/section/commands.html @@ -0,0 +1,33 @@ +{{ template "chrome/header.html" . }} + +{{ template "chrome/navbar.html" . }} +
    +
    +
    +
    +

    {{ .Title }}
    {{ .Description }}

    +
    + + +

    Rclone Commands

    +

    This is an index of all commands in rclone.

    + +

    Docs autogenerated by Cobra.

    + + + +
    +
    + + +
    + {{ template "chrome/menu.html" . }} +
    +
    +{{ template "chrome/footer.copyright.html" . }} +
    +{{ template "chrome/footer.html" . }} diff --git a/.rclone_repo/docs/layouts/shortcodes/cdownload.html b/.rclone_repo/docs/layouts/shortcodes/cdownload.html new file mode 100755 index 0000000..573124a --- /dev/null +++ b/.rclone_repo/docs/layouts/shortcodes/cdownload.html @@ -0,0 +1 @@ + diff --git a/.rclone_repo/docs/layouts/shortcodes/download.html b/.rclone_repo/docs/layouts/shortcodes/download.html new file mode 100755 index 0000000..7c44c27 --- /dev/null +++ b/.rclone_repo/docs/layouts/shortcodes/download.html @@ -0,0 +1 @@ + diff --git a/.rclone_repo/docs/layouts/shortcodes/provider.html b/.rclone_repo/docs/layouts/shortcodes/provider.html new file mode 100755 index 0000000..8933f3d --- /dev/null +++ b/.rclone_repo/docs/layouts/shortcodes/provider.html @@ -0,0 +1 @@ +{{ .Get "name" }} diff --git a/.rclone_repo/docs/layouts/shortcodes/version.html b/.rclone_repo/docs/layouts/shortcodes/version.html new file mode 100755 index 0000000..8476e6e --- /dev/null +++ b/.rclone_repo/docs/layouts/shortcodes/version.html @@ -0,0 +1 @@ +{{ partial "version.html" . }} diff --git a/.rclone_repo/docs/layouts/sitemap.xml b/.rclone_repo/docs/layouts/sitemap.xml new file mode 100755 index 0000000..8eb623b --- /dev/null +++ b/.rclone_repo/docs/layouts/sitemap.xml @@ -0,0 +1,10 @@ + + {{ range .Data.Pages }} + + {{ .Permalink }} + {{ safeHTML ( .Date.Format "2006-01-02T15:04:05-07:00" ) }}{{ with .Sitemap.ChangeFreq }} + {{ . }}{{ end }}{{ if ge .Sitemap.Priority 0.0 }} + {{ .Sitemap.Priority }}{{ end }} + + {{ end }} + diff --git a/.rclone_repo/docs/static/css/bootstrap.css b/.rclone_repo/docs/static/css/bootstrap.css new file mode 100755 index 0000000..7b6fd6d --- /dev/null +++ b/.rclone_repo/docs/static/css/bootstrap.css @@ -0,0 +1,7 @@ +@import url("//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,700");/*! + * Bootswatch v3.1.1 + * Homepage: http://bootswatch.com + * Copyright 2012-2014 Thomas Park + * Licensed under MIT + * Based on Bootstrap +*//*! normalize.css v3.0.0 | MIT License | git.io/normalize */html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}article,aside,details,figcaption,figure,footer,header,hgroup,main,nav,section,summary{display:block}audio,canvas,progress,video{display:inline-block;vertical-align:baseline}audio:not([controls]){display:none;height:0}[hidden],template{display:none}a{background:transparent}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:bold}dfn{font-style:italic}h1{font-size:2em;margin:0.67em 0}mark{background:#ff0;color:#000}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{border:0}svg:not(:root){overflow:hidden}figure{margin:1em 40px}hr{-moz-box-sizing:content-box;box-sizing:content-box;height:0}pre{overflow:auto}code,kbd,pre,samp{font-family:monospace, monospace;font-size:1em}button,input,optgroup,select,textarea{color:inherit;font:inherit;margin:0}button{overflow:visible}button,select{text-transform:none}button,html input[type="button"],input[type="reset"],input[type="submit"]{-webkit-appearance:button;cursor:pointer}button[disabled],html input[disabled]{cursor:default}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}input{line-height:normal}input[type="checkbox"],input[type="radio"]{box-sizing:border-box;padding:0}input[type="number"]::-webkit-inner-spin-button,input[type="number"]::-webkit-outer-spin-button{height:auto}input[type="search"]{-webkit-appearance:textfield;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;box-sizing:content-box}input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-decoration{-webkit-appearance:none}fieldset{border:1px solid #c0c0c0;margin:0 2px;padding:0.35em 0.625em 0.75em}legend{border:0;padding:0}textarea{overflow:auto}optgroup{font-weight:bold}table{border-collapse:collapse;border-spacing:0}td,th{padding:0}@media print{*{text-shadow:none !important;color:#000 !important;background:transparent !important;box-shadow:none !important}a,a:visited{text-decoration:underline}a[href]:after{content:" (" attr(href) ")"}abbr[title]:after{content:" (" attr(title) ")"}a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}p,h2,h3{orphans:3;widows:3}h2,h3{page-break-after:avoid}select{background:#fff !important}.navbar{display:none}.table td,.table th{background-color:#fff !important}.btn>.caret,.dropup>.btn>.caret{border-top-color:#000 !important}.label{border:1px solid #000}.table{border-collapse:collapse !important}.table-bordered th,.table-bordered td{border:1px solid #ddd !important}}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}*:before,*:after{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}html{font-size:62.5%;-webkit-tap-highlight-color:rgba(0,0,0,0)}body{font-family:"Open Sans","Helvetica Neue",Helvetica,Arial,sans-serif;font-size:15px;line-height:1.42857143;color:#222222;background-color:#ffffff}input,button,select,textarea{font-family:inherit;font-size:inherit;line-height:inherit}a{color:#008cba;text-decoration:none}a:hover,a:focus{color:#00526e;text-decoration:underline}a:focus{outline:thin dotted;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}figure{margin:0}img{vertical-align:middle}.img-responsive,.thumbnail>img,.thumbnail a>img,.carousel-inner>.item>img,.carousel-inner>.item>a>img{display:block;max-width:100%;height:auto}.img-rounded{border-radius:0}.img-thumbnail{padding:4px;line-height:1.42857143;background-color:#ffffff;border:1px solid #dddddd;border-radius:0;-webkit-transition:all .2s ease-in-out;transition:all .2s ease-in-out;display:inline-block;max-width:100%;height:auto}.img-circle{border-radius:50%}hr{margin-top:21px;margin-bottom:21px;border:0;border-top:1px solid #dddddd}.sr-only{position:absolute;width:1px;height:1px;margin:-1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);border:0}h1,h2,h3,h4,h5,h6,.h1,.h2,.h3,.h4,.h5,.h6{font-family:"Open Sans","Helvetica Neue",Helvetica,Arial,sans-serif;font-weight:300;line-height:1.1;color:inherit}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small,.h1 small,.h2 small,.h3 small,.h4 small,.h5 small,.h6 small,h1 .small,h2 .small,h3 .small,h4 .small,h5 .small,h6 .small,.h1 .small,.h2 .small,.h3 .small,.h4 .small,.h5 .small,.h6 .small{font-weight:normal;line-height:1;color:#999999}h1,.h1,h2,.h2,h3,.h3{margin-top:21px;margin-bottom:10.5px}h1 small,.h1 small,h2 small,.h2 small,h3 small,.h3 small,h1 .small,.h1 .small,h2 .small,.h2 .small,h3 .small,.h3 .small{font-size:65%}h4,.h4,h5,.h5,h6,.h6{margin-top:10.5px;margin-bottom:10.5px}h4 small,.h4 small,h5 small,.h5 small,h6 small,.h6 small,h4 .small,.h4 .small,h5 .small,.h5 .small,h6 .small,.h6 .small{font-size:75%}h1,.h1{font-size:39px}h2,.h2{font-size:32px}h3,.h3{font-size:26px}h4,.h4{font-size:19px}h5,.h5{font-size:15px}h6,.h6{font-size:13px}p{margin:0 0 10.5px}.lead{margin-bottom:21px;font-size:17px;font-weight:200;line-height:1.4}@media (min-width:768px){.lead{font-size:22.5px}}small,.small{font-size:85%}cite{font-style:normal}.text-left{text-align:left}.text-right{text-align:right}.text-center{text-align:center}.text-justify{text-align:justify}.text-muted{color:#999999}.text-primary{color:#008cba}a.text-primary:hover{color:#006687}.text-success{color:#43ac6a}a.text-success:hover{color:#358753}.text-info{color:#5bc0de}a.text-info:hover{color:#31b0d5}.text-warning{color:#e99002}a.text-warning:hover{color:#b67102}.text-danger{color:#f04124}a.text-danger:hover{color:#d32a0e}.bg-primary{color:#fff;background-color:#008cba}a.bg-primary:hover{background-color:#006687}.bg-success{background-color:#dff0d8}a.bg-success:hover{background-color:#c1e2b3}.bg-info{background-color:#d9edf7}a.bg-info:hover{background-color:#afd9ee}.bg-warning{background-color:#fcf8e3}a.bg-warning:hover{background-color:#f7ecb5}.bg-danger{background-color:#f2dede}a.bg-danger:hover{background-color:#e4b9b9}.page-header{padding-bottom:9.5px;margin:42px 0 21px;border-bottom:1px solid #dddddd}ul,ol{margin-top:0;margin-bottom:10.5px}ul ul,ol ul,ul ol,ol ol{margin-bottom:0}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none;margin-left:-5px}.list-inline>li{display:inline-block;padding-left:5px;padding-right:5px}dl{margin-top:0;margin-bottom:21px}dt,dd{line-height:1.42857143}dt{font-weight:bold}dd{margin-left:0}@media (min-width:768px){.dl-horizontal dt{float:left;width:160px;clear:left;text-align:right;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:180px}}abbr[title],abbr[data-original-title]{cursor:help;border-bottom:1px dotted #999999}.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:10.5px 21px;margin:0 0 21px;font-size:18.75px;border-left:5px solid #dddddd}blockquote p:last-child,blockquote ul:last-child,blockquote ol:last-child{margin-bottom:0}blockquote footer,blockquote small,blockquote .small{display:block;font-size:80%;line-height:1.42857143;color:#6f6f6f}blockquote footer:before,blockquote small:before,blockquote .small:before{content:'\2014 \00A0'}.blockquote-reverse,blockquote.pull-right{padding-right:15px;padding-left:0;border-right:5px solid #dddddd;border-left:0;text-align:right}.blockquote-reverse footer:before,blockquote.pull-right footer:before,.blockquote-reverse small:before,blockquote.pull-right small:before,.blockquote-reverse .small:before,blockquote.pull-right .small:before{content:''}.blockquote-reverse footer:after,blockquote.pull-right footer:after,.blockquote-reverse small:after,blockquote.pull-right small:after,.blockquote-reverse .small:after,blockquote.pull-right .small:after{content:'\00A0 \2014'}blockquote:before,blockquote:after{content:""}address{margin-bottom:21px;font-style:normal;line-height:1.42857143}code,kbd,pre,samp{font-family:Menlo,Monaco,Consolas,"Courier New",monospace}code{padding:2px 4px;font-size:90%;color:#c7254e;background-color:#f9f2f4;white-space:nowrap;border-radius:0}kbd{padding:2px 4px;font-size:90%;color:#ffffff;background-color:#333333;border-radius:0;box-shadow:inset 0 -1px 0 rgba(0,0,0,0.25)}pre{display:block;padding:10px;margin:0 0 10.5px;font-size:14px;line-height:1.42857143;word-break:break-all;word-wrap:break-word;color:#333333;background-color:#f5f5f5;border:1px solid #cccccc;border-radius:0}pre code{padding:0;font-size:inherit;color:inherit;white-space:pre-wrap;background-color:transparent;border-radius:0}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{margin-right:auto;margin-left:auto;padding-left:15px;padding-right:15px}@media (min-width:768px){.container{width:750px}}@media (min-width:992px){.container{width:970px}}@media (min-width:1200px){.container{width:1170px}}.container-fluid{margin-right:auto;margin-left:auto;padding-left:15px;padding-right:15px}.row{margin-left:-15px;margin-right:-15px}.col-xs-1,.col-sm-1,.col-md-1,.col-lg-1,.col-xs-2,.col-sm-2,.col-md-2,.col-lg-2,.col-xs-3,.col-sm-3,.col-md-3,.col-lg-3,.col-xs-4,.col-sm-4,.col-md-4,.col-lg-4,.col-xs-5,.col-sm-5,.col-md-5,.col-lg-5,.col-xs-6,.col-sm-6,.col-md-6,.col-lg-6,.col-xs-7,.col-sm-7,.col-md-7,.col-lg-7,.col-xs-8,.col-sm-8,.col-md-8,.col-lg-8,.col-xs-9,.col-sm-9,.col-md-9,.col-lg-9,.col-xs-10,.col-sm-10,.col-md-10,.col-lg-10,.col-xs-11,.col-sm-11,.col-md-11,.col-lg-11,.col-xs-12,.col-sm-12,.col-md-12,.col-lg-12{position:relative;min-height:1px;padding-left:15px;padding-right:15px}.col-xs-1,.col-xs-2,.col-xs-3,.col-xs-4,.col-xs-5,.col-xs-6,.col-xs-7,.col-xs-8,.col-xs-9,.col-xs-10,.col-xs-11,.col-xs-12{float:left}.col-xs-12{width:100%}.col-xs-11{width:91.66666667%}.col-xs-10{width:83.33333333%}.col-xs-9{width:75%}.col-xs-8{width:66.66666667%}.col-xs-7{width:58.33333333%}.col-xs-6{width:50%}.col-xs-5{width:41.66666667%}.col-xs-4{width:33.33333333%}.col-xs-3{width:25%}.col-xs-2{width:16.66666667%}.col-xs-1{width:8.33333333%}.col-xs-pull-12{right:100%}.col-xs-pull-11{right:91.66666667%}.col-xs-pull-10{right:83.33333333%}.col-xs-pull-9{right:75%}.col-xs-pull-8{right:66.66666667%}.col-xs-pull-7{right:58.33333333%}.col-xs-pull-6{right:50%}.col-xs-pull-5{right:41.66666667%}.col-xs-pull-4{right:33.33333333%}.col-xs-pull-3{right:25%}.col-xs-pull-2{right:16.66666667%}.col-xs-pull-1{right:8.33333333%}.col-xs-pull-0{right:0%}.col-xs-push-12{left:100%}.col-xs-push-11{left:91.66666667%}.col-xs-push-10{left:83.33333333%}.col-xs-push-9{left:75%}.col-xs-push-8{left:66.66666667%}.col-xs-push-7{left:58.33333333%}.col-xs-push-6{left:50%}.col-xs-push-5{left:41.66666667%}.col-xs-push-4{left:33.33333333%}.col-xs-push-3{left:25%}.col-xs-push-2{left:16.66666667%}.col-xs-push-1{left:8.33333333%}.col-xs-push-0{left:0%}.col-xs-offset-12{margin-left:100%}.col-xs-offset-11{margin-left:91.66666667%}.col-xs-offset-10{margin-left:83.33333333%}.col-xs-offset-9{margin-left:75%}.col-xs-offset-8{margin-left:66.66666667%}.col-xs-offset-7{margin-left:58.33333333%}.col-xs-offset-6{margin-left:50%}.col-xs-offset-5{margin-left:41.66666667%}.col-xs-offset-4{margin-left:33.33333333%}.col-xs-offset-3{margin-left:25%}.col-xs-offset-2{margin-left:16.66666667%}.col-xs-offset-1{margin-left:8.33333333%}.col-xs-offset-0{margin-left:0%}@media (min-width:768px){.col-sm-1,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-sm-10,.col-sm-11,.col-sm-12{float:left}.col-sm-12{width:100%}.col-sm-11{width:91.66666667%}.col-sm-10{width:83.33333333%}.col-sm-9{width:75%}.col-sm-8{width:66.66666667%}.col-sm-7{width:58.33333333%}.col-sm-6{width:50%}.col-sm-5{width:41.66666667%}.col-sm-4{width:33.33333333%}.col-sm-3{width:25%}.col-sm-2{width:16.66666667%}.col-sm-1{width:8.33333333%}.col-sm-pull-12{right:100%}.col-sm-pull-11{right:91.66666667%}.col-sm-pull-10{right:83.33333333%}.col-sm-pull-9{right:75%}.col-sm-pull-8{right:66.66666667%}.col-sm-pull-7{right:58.33333333%}.col-sm-pull-6{right:50%}.col-sm-pull-5{right:41.66666667%}.col-sm-pull-4{right:33.33333333%}.col-sm-pull-3{right:25%}.col-sm-pull-2{right:16.66666667%}.col-sm-pull-1{right:8.33333333%}.col-sm-pull-0{right:0%}.col-sm-push-12{left:100%}.col-sm-push-11{left:91.66666667%}.col-sm-push-10{left:83.33333333%}.col-sm-push-9{left:75%}.col-sm-push-8{left:66.66666667%}.col-sm-push-7{left:58.33333333%}.col-sm-push-6{left:50%}.col-sm-push-5{left:41.66666667%}.col-sm-push-4{left:33.33333333%}.col-sm-push-3{left:25%}.col-sm-push-2{left:16.66666667%}.col-sm-push-1{left:8.33333333%}.col-sm-push-0{left:0%}.col-sm-offset-12{margin-left:100%}.col-sm-offset-11{margin-left:91.66666667%}.col-sm-offset-10{margin-left:83.33333333%}.col-sm-offset-9{margin-left:75%}.col-sm-offset-8{margin-left:66.66666667%}.col-sm-offset-7{margin-left:58.33333333%}.col-sm-offset-6{margin-left:50%}.col-sm-offset-5{margin-left:41.66666667%}.col-sm-offset-4{margin-left:33.33333333%}.col-sm-offset-3{margin-left:25%}.col-sm-offset-2{margin-left:16.66666667%}.col-sm-offset-1{margin-left:8.33333333%}.col-sm-offset-0{margin-left:0%}}@media (min-width:992px){.col-md-1,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-md-10,.col-md-11,.col-md-12{float:left}.col-md-12{width:100%}.col-md-11{width:91.66666667%}.col-md-10{width:83.33333333%}.col-md-9{width:75%}.col-md-8{width:66.66666667%}.col-md-7{width:58.33333333%}.col-md-6{width:50%}.col-md-5{width:41.66666667%}.col-md-4{width:33.33333333%}.col-md-3{width:25%}.col-md-2{width:16.66666667%}.col-md-1{width:8.33333333%}.col-md-pull-12{right:100%}.col-md-pull-11{right:91.66666667%}.col-md-pull-10{right:83.33333333%}.col-md-pull-9{right:75%}.col-md-pull-8{right:66.66666667%}.col-md-pull-7{right:58.33333333%}.col-md-pull-6{right:50%}.col-md-pull-5{right:41.66666667%}.col-md-pull-4{right:33.33333333%}.col-md-pull-3{right:25%}.col-md-pull-2{right:16.66666667%}.col-md-pull-1{right:8.33333333%}.col-md-pull-0{right:0%}.col-md-push-12{left:100%}.col-md-push-11{left:91.66666667%}.col-md-push-10{left:83.33333333%}.col-md-push-9{left:75%}.col-md-push-8{left:66.66666667%}.col-md-push-7{left:58.33333333%}.col-md-push-6{left:50%}.col-md-push-5{left:41.66666667%}.col-md-push-4{left:33.33333333%}.col-md-push-3{left:25%}.col-md-push-2{left:16.66666667%}.col-md-push-1{left:8.33333333%}.col-md-push-0{left:0%}.col-md-offset-12{margin-left:100%}.col-md-offset-11{margin-left:91.66666667%}.col-md-offset-10{margin-left:83.33333333%}.col-md-offset-9{margin-left:75%}.col-md-offset-8{margin-left:66.66666667%}.col-md-offset-7{margin-left:58.33333333%}.col-md-offset-6{margin-left:50%}.col-md-offset-5{margin-left:41.66666667%}.col-md-offset-4{margin-left:33.33333333%}.col-md-offset-3{margin-left:25%}.col-md-offset-2{margin-left:16.66666667%}.col-md-offset-1{margin-left:8.33333333%}.col-md-offset-0{margin-left:0%}}@media (min-width:1200px){.col-lg-1,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-lg-10,.col-lg-11,.col-lg-12{float:left}.col-lg-12{width:100%}.col-lg-11{width:91.66666667%}.col-lg-10{width:83.33333333%}.col-lg-9{width:75%}.col-lg-8{width:66.66666667%}.col-lg-7{width:58.33333333%}.col-lg-6{width:50%}.col-lg-5{width:41.66666667%}.col-lg-4{width:33.33333333%}.col-lg-3{width:25%}.col-lg-2{width:16.66666667%}.col-lg-1{width:8.33333333%}.col-lg-pull-12{right:100%}.col-lg-pull-11{right:91.66666667%}.col-lg-pull-10{right:83.33333333%}.col-lg-pull-9{right:75%}.col-lg-pull-8{right:66.66666667%}.col-lg-pull-7{right:58.33333333%}.col-lg-pull-6{right:50%}.col-lg-pull-5{right:41.66666667%}.col-lg-pull-4{right:33.33333333%}.col-lg-pull-3{right:25%}.col-lg-pull-2{right:16.66666667%}.col-lg-pull-1{right:8.33333333%}.col-lg-pull-0{right:0%}.col-lg-push-12{left:100%}.col-lg-push-11{left:91.66666667%}.col-lg-push-10{left:83.33333333%}.col-lg-push-9{left:75%}.col-lg-push-8{left:66.66666667%}.col-lg-push-7{left:58.33333333%}.col-lg-push-6{left:50%}.col-lg-push-5{left:41.66666667%}.col-lg-push-4{left:33.33333333%}.col-lg-push-3{left:25%}.col-lg-push-2{left:16.66666667%}.col-lg-push-1{left:8.33333333%}.col-lg-push-0{left:0%}.col-lg-offset-12{margin-left:100%}.col-lg-offset-11{margin-left:91.66666667%}.col-lg-offset-10{margin-left:83.33333333%}.col-lg-offset-9{margin-left:75%}.col-lg-offset-8{margin-left:66.66666667%}.col-lg-offset-7{margin-left:58.33333333%}.col-lg-offset-6{margin-left:50%}.col-lg-offset-5{margin-left:41.66666667%}.col-lg-offset-4{margin-left:33.33333333%}.col-lg-offset-3{margin-left:25%}.col-lg-offset-2{margin-left:16.66666667%}.col-lg-offset-1{margin-left:8.33333333%}.col-lg-offset-0{margin-left:0%}}table{max-width:100%;background-color:transparent}th{text-align:left}.table{width:100%;margin-bottom:21px}.table>thead>tr>th,.table>tbody>tr>th,.table>tfoot>tr>th,.table>thead>tr>td,.table>tbody>tr>td,.table>tfoot>tr>td{padding:8px;line-height:1.42857143;vertical-align:top;border-top:1px solid #dddddd}.table>thead>tr>th{vertical-align:bottom;border-bottom:2px solid #dddddd}.table>caption+thead>tr:first-child>th,.table>colgroup+thead>tr:first-child>th,.table>thead:first-child>tr:first-child>th,.table>caption+thead>tr:first-child>td,.table>colgroup+thead>tr:first-child>td,.table>thead:first-child>tr:first-child>td{border-top:0}.table>tbody+tbody{border-top:2px solid #dddddd}.table .table{background-color:#ffffff}.table-condensed>thead>tr>th,.table-condensed>tbody>tr>th,.table-condensed>tfoot>tr>th,.table-condensed>thead>tr>td,.table-condensed>tbody>tr>td,.table-condensed>tfoot>tr>td{padding:5px}.table-bordered{border:1px solid #dddddd}.table-bordered>thead>tr>th,.table-bordered>tbody>tr>th,.table-bordered>tfoot>tr>th,.table-bordered>thead>tr>td,.table-bordered>tbody>tr>td,.table-bordered>tfoot>tr>td{border:1px solid #dddddd}.table-bordered>thead>tr>th,.table-bordered>thead>tr>td{border-bottom-width:2px}.table-striped>tbody>tr:nth-child(odd)>td,.table-striped>tbody>tr:nth-child(odd)>th{background-color:#f9f9f9}.table-hover>tbody>tr:hover>td,.table-hover>tbody>tr:hover>th{background-color:#f5f5f5}table col[class*="col-"]{position:static;float:none;display:table-column}table td[class*="col-"],table th[class*="col-"]{position:static;float:none;display:table-cell}.table>thead>tr>td.active,.table>tbody>tr>td.active,.table>tfoot>tr>td.active,.table>thead>tr>th.active,.table>tbody>tr>th.active,.table>tfoot>tr>th.active,.table>thead>tr.active>td,.table>tbody>tr.active>td,.table>tfoot>tr.active>td,.table>thead>tr.active>th,.table>tbody>tr.active>th,.table>tfoot>tr.active>th{background-color:#f5f5f5}.table-hover>tbody>tr>td.active:hover,.table-hover>tbody>tr>th.active:hover,.table-hover>tbody>tr.active:hover>td,.table-hover>tbody>tr.active:hover>th{background-color:#e8e8e8}.table>thead>tr>td.success,.table>tbody>tr>td.success,.table>tfoot>tr>td.success,.table>thead>tr>th.success,.table>tbody>tr>th.success,.table>tfoot>tr>th.success,.table>thead>tr.success>td,.table>tbody>tr.success>td,.table>tfoot>tr.success>td,.table>thead>tr.success>th,.table>tbody>tr.success>th,.table>tfoot>tr.success>th{background-color:#dff0d8}.table-hover>tbody>tr>td.success:hover,.table-hover>tbody>tr>th.success:hover,.table-hover>tbody>tr.success:hover>td,.table-hover>tbody>tr.success:hover>th{background-color:#d0e9c6}.table>thead>tr>td.info,.table>tbody>tr>td.info,.table>tfoot>tr>td.info,.table>thead>tr>th.info,.table>tbody>tr>th.info,.table>tfoot>tr>th.info,.table>thead>tr.info>td,.table>tbody>tr.info>td,.table>tfoot>tr.info>td,.table>thead>tr.info>th,.table>tbody>tr.info>th,.table>tfoot>tr.info>th{background-color:#d9edf7}.table-hover>tbody>tr>td.info:hover,.table-hover>tbody>tr>th.info:hover,.table-hover>tbody>tr.info:hover>td,.table-hover>tbody>tr.info:hover>th{background-color:#c4e3f3}.table>thead>tr>td.warning,.table>tbody>tr>td.warning,.table>tfoot>tr>td.warning,.table>thead>tr>th.warning,.table>tbody>tr>th.warning,.table>tfoot>tr>th.warning,.table>thead>tr.warning>td,.table>tbody>tr.warning>td,.table>tfoot>tr.warning>td,.table>thead>tr.warning>th,.table>tbody>tr.warning>th,.table>tfoot>tr.warning>th{background-color:#fcf8e3}.table-hover>tbody>tr>td.warning:hover,.table-hover>tbody>tr>th.warning:hover,.table-hover>tbody>tr.warning:hover>td,.table-hover>tbody>tr.warning:hover>th{background-color:#faf2cc}.table>thead>tr>td.danger,.table>tbody>tr>td.danger,.table>tfoot>tr>td.danger,.table>thead>tr>th.danger,.table>tbody>tr>th.danger,.table>tfoot>tr>th.danger,.table>thead>tr.danger>td,.table>tbody>tr.danger>td,.table>tfoot>tr.danger>td,.table>thead>tr.danger>th,.table>tbody>tr.danger>th,.table>tfoot>tr.danger>th{background-color:#f2dede}.table-hover>tbody>tr>td.danger:hover,.table-hover>tbody>tr>th.danger:hover,.table-hover>tbody>tr.danger:hover>td,.table-hover>tbody>tr.danger:hover>th{background-color:#ebcccc}@media (max-width:767px){.table-responsive{width:100%;margin-bottom:15.75px;overflow-y:hidden;overflow-x:scroll;-ms-overflow-style:-ms-autohiding-scrollbar;border:1px solid #dddddd;-webkit-overflow-scrolling:touch}.table-responsive>.table{margin-bottom:0}.table-responsive>.table>thead>tr>th,.table-responsive>.table>tbody>tr>th,.table-responsive>.table>tfoot>tr>th,.table-responsive>.table>thead>tr>td,.table-responsive>.table>tbody>tr>td,.table-responsive>.table>tfoot>tr>td{white-space:nowrap}.table-responsive>.table-bordered{border:0}.table-responsive>.table-bordered>thead>tr>th:first-child,.table-responsive>.table-bordered>tbody>tr>th:first-child,.table-responsive>.table-bordered>tfoot>tr>th:first-child,.table-responsive>.table-bordered>thead>tr>td:first-child,.table-responsive>.table-bordered>tbody>tr>td:first-child,.table-responsive>.table-bordered>tfoot>tr>td:first-child{border-left:0}.table-responsive>.table-bordered>thead>tr>th:last-child,.table-responsive>.table-bordered>tbody>tr>th:last-child,.table-responsive>.table-bordered>tfoot>tr>th:last-child,.table-responsive>.table-bordered>thead>tr>td:last-child,.table-responsive>.table-bordered>tbody>tr>td:last-child,.table-responsive>.table-bordered>tfoot>tr>td:last-child{border-right:0}.table-responsive>.table-bordered>tbody>tr:last-child>th,.table-responsive>.table-bordered>tfoot>tr:last-child>th,.table-responsive>.table-bordered>tbody>tr:last-child>td,.table-responsive>.table-bordered>tfoot>tr:last-child>td{border-bottom:0}}fieldset{padding:0;margin:0;border:0;min-width:0}legend{display:block;width:100%;padding:0;margin-bottom:21px;font-size:22.5px;line-height:inherit;color:#333333;border:0;border-bottom:1px solid #e5e5e5}label{display:inline-block;margin-bottom:5px;font-weight:bold}input[type="search"]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type="radio"],input[type="checkbox"]{margin:4px 0 0;margin-top:1px \9;line-height:normal}input[type="file"]{display:block}input[type="range"]{display:block;width:100%}select[multiple],select[size]{height:auto}input[type="file"]:focus,input[type="radio"]:focus,input[type="checkbox"]:focus{outline:thin dotted;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}output{display:block;padding-top:7px;font-size:15px;line-height:1.42857143;color:#6f6f6f}.form-control{display:block;width:100%;height:35px;padding:6px 12px;font-size:15px;line-height:1.42857143;color:#6f6f6f;background-color:#ffffff;background-image:none;border:1px solid #cccccc;border-radius:0;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-webkit-transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s}.form-control:focus{border-color:#66afe9;outline:0;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(102,175,233,0.6);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(102,175,233,0.6)}.form-control::-moz-placeholder{color:#999999;opacity:1}.form-control:-ms-input-placeholder{color:#999999}.form-control::-webkit-input-placeholder{color:#999999}.form-control[disabled],.form-control[readonly],fieldset[disabled] .form-control{cursor:not-allowed;background-color:#eeeeee;opacity:1}textarea.form-control{height:auto}input[type="search"]{-webkit-appearance:none}input[type="date"]{line-height:35px}.form-group{margin-bottom:15px}.radio,.checkbox{display:block;min-height:21px;margin-top:10px;margin-bottom:10px;padding-left:20px}.radio label,.checkbox label{display:inline;font-weight:normal;cursor:pointer}.radio input[type="radio"],.radio-inline input[type="radio"],.checkbox input[type="checkbox"],.checkbox-inline input[type="checkbox"]{float:left;margin-left:-20px}.radio+.radio,.checkbox+.checkbox{margin-top:-5px}.radio-inline,.checkbox-inline{display:inline-block;padding-left:20px;margin-bottom:0;vertical-align:middle;font-weight:normal;cursor:pointer}.radio-inline+.radio-inline,.checkbox-inline+.checkbox-inline{margin-top:0;margin-left:10px}input[type="radio"][disabled],input[type="checkbox"][disabled],.radio[disabled],.radio-inline[disabled],.checkbox[disabled],.checkbox-inline[disabled],fieldset[disabled] input[type="radio"],fieldset[disabled] input[type="checkbox"],fieldset[disabled] .radio,fieldset[disabled] .radio-inline,fieldset[disabled] .checkbox,fieldset[disabled] .checkbox-inline{cursor:not-allowed}.input-sm{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:0}select.input-sm{height:30px;line-height:30px}textarea.input-sm,select[multiple].input-sm{height:auto}.input-lg{height:48px;padding:10px 16px;font-size:19px;line-height:1.33;border-radius:0}select.input-lg{height:48px;line-height:48px}textarea.input-lg,select[multiple].input-lg{height:auto}.has-feedback{position:relative}.has-feedback .form-control{padding-right:43.75px}.has-feedback .form-control-feedback{position:absolute;top:26px;right:0;display:block;width:35px;height:35px;line-height:35px;text-align:center}.has-success .help-block,.has-success .control-label,.has-success .radio,.has-success .checkbox,.has-success .radio-inline,.has-success .checkbox-inline{color:#43ac6a}.has-success .form-control{border-color:#43ac6a;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.has-success .form-control:focus{border-color:#358753;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #85d0a1;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #85d0a1}.has-success .input-group-addon{color:#43ac6a;border-color:#43ac6a;background-color:#dff0d8}.has-success .form-control-feedback{color:#43ac6a}.has-warning .help-block,.has-warning .control-label,.has-warning .radio,.has-warning .checkbox,.has-warning .radio-inline,.has-warning .checkbox-inline{color:#e99002}.has-warning .form-control{border-color:#e99002;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.has-warning .form-control:focus{border-color:#b67102;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #febc53;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #febc53}.has-warning .input-group-addon{color:#e99002;border-color:#e99002;background-color:#fcf8e3}.has-warning .form-control-feedback{color:#e99002}.has-error .help-block,.has-error .control-label,.has-error .radio,.has-error .checkbox,.has-error .radio-inline,.has-error .checkbox-inline{color:#f04124}.has-error .form-control{border-color:#f04124;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.has-error .form-control:focus{border-color:#d32a0e;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #f79483;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #f79483}.has-error .input-group-addon{color:#f04124;border-color:#f04124;background-color:#f2dede}.has-error .form-control-feedback{color:#f04124}.form-control-static{margin-bottom:0}.help-block{display:block;margin-top:5px;margin-bottom:10px;color:#626262}@media (min-width:768px){.form-inline .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .input-group>.form-control{width:100%}.form-inline .control-label{margin-bottom:0;vertical-align:middle}.form-inline .radio,.form-inline .checkbox{display:inline-block;margin-top:0;margin-bottom:0;padding-left:0;vertical-align:middle}.form-inline .radio input[type="radio"],.form-inline .checkbox input[type="checkbox"]{float:none;margin-left:0}.form-inline .has-feedback .form-control-feedback{top:0}}.form-horizontal .control-label,.form-horizontal .radio,.form-horizontal .checkbox,.form-horizontal .radio-inline,.form-horizontal .checkbox-inline{margin-top:0;margin-bottom:0;padding-top:7px}.form-horizontal .radio,.form-horizontal .checkbox{min-height:28px}.form-horizontal .form-group{margin-left:-15px;margin-right:-15px}.form-horizontal .form-control-static{padding-top:7px}@media (min-width:768px){.form-horizontal .control-label{text-align:right}}.form-horizontal .has-feedback .form-control-feedback{top:0;right:15px}.btn{display:inline-block;margin-bottom:0;font-weight:normal;text-align:center;vertical-align:middle;cursor:pointer;background-image:none;border:1px solid transparent;white-space:nowrap;padding:6px 12px;font-size:15px;line-height:1.42857143;border-radius:0;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.btn:focus,.btn:active:focus,.btn.active:focus{outline:thin dotted;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn:hover,.btn:focus{color:#333333;text-decoration:none}.btn:active,.btn.active{outline:0;background-image:none;-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,0.125);box-shadow:inset 0 3px 5px rgba(0,0,0,0.125)}.btn.disabled,.btn[disabled],fieldset[disabled] .btn{cursor:not-allowed;pointer-events:none;opacity:0.65;filter:alpha(opacity=65);-webkit-box-shadow:none;box-shadow:none}.btn-default{color:#333333;background-color:#e7e7e7;border-color:#dadada}.btn-default:hover,.btn-default:focus,.btn-default:active,.btn-default.active,.open .dropdown-toggle.btn-default{color:#333333;background-color:#d3d3d3;border-color:#bbbbbb}.btn-default:active,.btn-default.active,.open .dropdown-toggle.btn-default{background-image:none}.btn-default.disabled,.btn-default[disabled],fieldset[disabled] .btn-default,.btn-default.disabled:hover,.btn-default[disabled]:hover,fieldset[disabled] .btn-default:hover,.btn-default.disabled:focus,.btn-default[disabled]:focus,fieldset[disabled] .btn-default:focus,.btn-default.disabled:active,.btn-default[disabled]:active,fieldset[disabled] .btn-default:active,.btn-default.disabled.active,.btn-default[disabled].active,fieldset[disabled] .btn-default.active{background-color:#e7e7e7;border-color:#dadada}.btn-default .badge{color:#e7e7e7;background-color:#333333}.btn-primary{color:#ffffff;background-color:#008cba;border-color:#0079a1}.btn-primary:hover,.btn-primary:focus,.btn-primary:active,.btn-primary.active,.open .dropdown-toggle.btn-primary{color:#ffffff;background-color:#006d91;border-color:#004b63}.btn-primary:active,.btn-primary.active,.open .dropdown-toggle.btn-primary{background-image:none}.btn-primary.disabled,.btn-primary[disabled],fieldset[disabled] .btn-primary,.btn-primary.disabled:hover,.btn-primary[disabled]:hover,fieldset[disabled] .btn-primary:hover,.btn-primary.disabled:focus,.btn-primary[disabled]:focus,fieldset[disabled] .btn-primary:focus,.btn-primary.disabled:active,.btn-primary[disabled]:active,fieldset[disabled] .btn-primary:active,.btn-primary.disabled.active,.btn-primary[disabled].active,fieldset[disabled] .btn-primary.active{background-color:#008cba;border-color:#0079a1}.btn-primary .badge{color:#008cba;background-color:#ffffff}.btn-success{color:#ffffff;background-color:#43ac6a;border-color:#3c9a5f}.btn-success:hover,.btn-success:focus,.btn-success:active,.btn-success.active,.open .dropdown-toggle.btn-success{color:#ffffff;background-color:#388f58;border-color:#2b6e44}.btn-success:active,.btn-success.active,.open .dropdown-toggle.btn-success{background-image:none}.btn-success.disabled,.btn-success[disabled],fieldset[disabled] .btn-success,.btn-success.disabled:hover,.btn-success[disabled]:hover,fieldset[disabled] .btn-success:hover,.btn-success.disabled:focus,.btn-success[disabled]:focus,fieldset[disabled] .btn-success:focus,.btn-success.disabled:active,.btn-success[disabled]:active,fieldset[disabled] .btn-success:active,.btn-success.disabled.active,.btn-success[disabled].active,fieldset[disabled] .btn-success.active{background-color:#43ac6a;border-color:#3c9a5f}.btn-success .badge{color:#43ac6a;background-color:#ffffff}.btn-info{color:#ffffff;background-color:#5bc0de;border-color:#46b8da}.btn-info:hover,.btn-info:focus,.btn-info:active,.btn-info.active,.open .dropdown-toggle.btn-info{color:#ffffff;background-color:#39b3d7;border-color:#269abc}.btn-info:active,.btn-info.active,.open .dropdown-toggle.btn-info{background-image:none}.btn-info.disabled,.btn-info[disabled],fieldset[disabled] .btn-info,.btn-info.disabled:hover,.btn-info[disabled]:hover,fieldset[disabled] .btn-info:hover,.btn-info.disabled:focus,.btn-info[disabled]:focus,fieldset[disabled] .btn-info:focus,.btn-info.disabled:active,.btn-info[disabled]:active,fieldset[disabled] .btn-info:active,.btn-info.disabled.active,.btn-info[disabled].active,fieldset[disabled] .btn-info.active{background-color:#5bc0de;border-color:#46b8da}.btn-info .badge{color:#5bc0de;background-color:#ffffff}.btn-warning{color:#ffffff;background-color:#e99002;border-color:#d08002}.btn-warning:hover,.btn-warning:focus,.btn-warning:active,.btn-warning.active,.open .dropdown-toggle.btn-warning{color:#ffffff;background-color:#c17702;border-color:#935b01}.btn-warning:active,.btn-warning.active,.open .dropdown-toggle.btn-warning{background-image:none}.btn-warning.disabled,.btn-warning[disabled],fieldset[disabled] .btn-warning,.btn-warning.disabled:hover,.btn-warning[disabled]:hover,fieldset[disabled] .btn-warning:hover,.btn-warning.disabled:focus,.btn-warning[disabled]:focus,fieldset[disabled] .btn-warning:focus,.btn-warning.disabled:active,.btn-warning[disabled]:active,fieldset[disabled] .btn-warning:active,.btn-warning.disabled.active,.btn-warning[disabled].active,fieldset[disabled] .btn-warning.active{background-color:#e99002;border-color:#d08002}.btn-warning .badge{color:#e99002;background-color:#ffffff}.btn-danger{color:#ffffff;background-color:#f04124;border-color:#ea2f10}.btn-danger:hover,.btn-danger:focus,.btn-danger:active,.btn-danger.active,.open .dropdown-toggle.btn-danger{color:#ffffff;background-color:#dc2c0f;border-color:#b1240c}.btn-danger:active,.btn-danger.active,.open .dropdown-toggle.btn-danger{background-image:none}.btn-danger.disabled,.btn-danger[disabled],fieldset[disabled] .btn-danger,.btn-danger.disabled:hover,.btn-danger[disabled]:hover,fieldset[disabled] .btn-danger:hover,.btn-danger.disabled:focus,.btn-danger[disabled]:focus,fieldset[disabled] .btn-danger:focus,.btn-danger.disabled:active,.btn-danger[disabled]:active,fieldset[disabled] .btn-danger:active,.btn-danger.disabled.active,.btn-danger[disabled].active,fieldset[disabled] .btn-danger.active{background-color:#f04124;border-color:#ea2f10}.btn-danger .badge{color:#f04124;background-color:#ffffff}.btn-link{color:#008cba;font-weight:normal;cursor:pointer;border-radius:0}.btn-link,.btn-link:active,.btn-link[disabled],fieldset[disabled] .btn-link{background-color:transparent;-webkit-box-shadow:none;box-shadow:none}.btn-link,.btn-link:hover,.btn-link:focus,.btn-link:active{border-color:transparent}.btn-link:hover,.btn-link:focus{color:#00526e;text-decoration:underline;background-color:transparent}.btn-link[disabled]:hover,fieldset[disabled] .btn-link:hover,.btn-link[disabled]:focus,fieldset[disabled] .btn-link:focus{color:#999999;text-decoration:none}.btn-lg,.btn-group-lg>.btn{padding:10px 16px;font-size:19px;line-height:1.33;border-radius:0}.btn-sm,.btn-group-sm>.btn{padding:5px 10px;font-size:12px;line-height:1.5;border-radius:0}.btn-xs,.btn-group-xs>.btn{padding:1px 5px;font-size:12px;line-height:1.5;border-radius:0}.btn-block{display:block;width:100%;padding-left:0;padding-right:0}.btn-block+.btn-block{margin-top:5px}input[type="submit"].btn-block,input[type="reset"].btn-block,input[type="button"].btn-block{width:100%}.fade{opacity:0;-webkit-transition:opacity 0.15s linear;transition:opacity 0.15s linear}.fade.in{opacity:1}.collapse{display:none}.collapse.in{display:block}.collapsing{position:relative;height:0;overflow:hidden;-webkit-transition:height 0.35s ease;transition:height 0.35s ease}@font-face{font-family:'Glyphicons Halflings';src:url('../fonts/glyphicons-halflings-regular.eot');src:url('../fonts/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'),url('../fonts/glyphicons-halflings-regular.woff') format('woff'),url('../fonts/glyphicons-halflings-regular.ttf') format('truetype'),url('../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg')}.glyphicon{position:relative;top:1px;display:inline-block;font-family:'Glyphicons Halflings';font-style:normal;font-weight:normal;line-height:1;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.glyphicon-asterisk:before{content:"\2a"}.glyphicon-plus:before{content:"\2b"}.glyphicon-euro:before{content:"\20ac"}.glyphicon-minus:before{content:"\2212"}.glyphicon-cloud:before{content:"\2601"}.glyphicon-envelope:before{content:"\2709"}.glyphicon-pencil:before{content:"\270f"}.glyphicon-glass:before{content:"\e001"}.glyphicon-music:before{content:"\e002"}.glyphicon-search:before{content:"\e003"}.glyphicon-heart:before{content:"\e005"}.glyphicon-star:before{content:"\e006"}.glyphicon-star-empty:before{content:"\e007"}.glyphicon-user:before{content:"\e008"}.glyphicon-film:before{content:"\e009"}.glyphicon-th-large:before{content:"\e010"}.glyphicon-th:before{content:"\e011"}.glyphicon-th-list:before{content:"\e012"}.glyphicon-ok:before{content:"\e013"}.glyphicon-remove:before{content:"\e014"}.glyphicon-zoom-in:before{content:"\e015"}.glyphicon-zoom-out:before{content:"\e016"}.glyphicon-off:before{content:"\e017"}.glyphicon-signal:before{content:"\e018"}.glyphicon-cog:before{content:"\e019"}.glyphicon-trash:before{content:"\e020"}.glyphicon-home:before{content:"\e021"}.glyphicon-file:before{content:"\e022"}.glyphicon-time:before{content:"\e023"}.glyphicon-road:before{content:"\e024"}.glyphicon-download-alt:before{content:"\e025"}.glyphicon-download:before{content:"\e026"}.glyphicon-upload:before{content:"\e027"}.glyphicon-inbox:before{content:"\e028"}.glyphicon-play-circle:before{content:"\e029"}.glyphicon-repeat:before{content:"\e030"}.glyphicon-refresh:before{content:"\e031"}.glyphicon-list-alt:before{content:"\e032"}.glyphicon-lock:before{content:"\e033"}.glyphicon-flag:before{content:"\e034"}.glyphicon-headphones:before{content:"\e035"}.glyphicon-volume-off:before{content:"\e036"}.glyphicon-volume-down:before{content:"\e037"}.glyphicon-volume-up:before{content:"\e038"}.glyphicon-qrcode:before{content:"\e039"}.glyphicon-barcode:before{content:"\e040"}.glyphicon-tag:before{content:"\e041"}.glyphicon-tags:before{content:"\e042"}.glyphicon-book:before{content:"\e043"}.glyphicon-bookmark:before{content:"\e044"}.glyphicon-print:before{content:"\e045"}.glyphicon-camera:before{content:"\e046"}.glyphicon-font:before{content:"\e047"}.glyphicon-bold:before{content:"\e048"}.glyphicon-italic:before{content:"\e049"}.glyphicon-text-height:before{content:"\e050"}.glyphicon-text-width:before{content:"\e051"}.glyphicon-align-left:before{content:"\e052"}.glyphicon-align-center:before{content:"\e053"}.glyphicon-align-right:before{content:"\e054"}.glyphicon-align-justify:before{content:"\e055"}.glyphicon-list:before{content:"\e056"}.glyphicon-indent-left:before{content:"\e057"}.glyphicon-indent-right:before{content:"\e058"}.glyphicon-facetime-video:before{content:"\e059"}.glyphicon-picture:before{content:"\e060"}.glyphicon-map-marker:before{content:"\e062"}.glyphicon-adjust:before{content:"\e063"}.glyphicon-tint:before{content:"\e064"}.glyphicon-edit:before{content:"\e065"}.glyphicon-share:before{content:"\e066"}.glyphicon-check:before{content:"\e067"}.glyphicon-move:before{content:"\e068"}.glyphicon-step-backward:before{content:"\e069"}.glyphicon-fast-backward:before{content:"\e070"}.glyphicon-backward:before{content:"\e071"}.glyphicon-play:before{content:"\e072"}.glyphicon-pause:before{content:"\e073"}.glyphicon-stop:before{content:"\e074"}.glyphicon-forward:before{content:"\e075"}.glyphicon-fast-forward:before{content:"\e076"}.glyphicon-step-forward:before{content:"\e077"}.glyphicon-eject:before{content:"\e078"}.glyphicon-chevron-left:before{content:"\e079"}.glyphicon-chevron-right:before{content:"\e080"}.glyphicon-plus-sign:before{content:"\e081"}.glyphicon-minus-sign:before{content:"\e082"}.glyphicon-remove-sign:before{content:"\e083"}.glyphicon-ok-sign:before{content:"\e084"}.glyphicon-question-sign:before{content:"\e085"}.glyphicon-info-sign:before{content:"\e086"}.glyphicon-screenshot:before{content:"\e087"}.glyphicon-remove-circle:before{content:"\e088"}.glyphicon-ok-circle:before{content:"\e089"}.glyphicon-ban-circle:before{content:"\e090"}.glyphicon-arrow-left:before{content:"\e091"}.glyphicon-arrow-right:before{content:"\e092"}.glyphicon-arrow-up:before{content:"\e093"}.glyphicon-arrow-down:before{content:"\e094"}.glyphicon-share-alt:before{content:"\e095"}.glyphicon-resize-full:before{content:"\e096"}.glyphicon-resize-small:before{content:"\e097"}.glyphicon-exclamation-sign:before{content:"\e101"}.glyphicon-gift:before{content:"\e102"}.glyphicon-leaf:before{content:"\e103"}.glyphicon-fire:before{content:"\e104"}.glyphicon-eye-open:before{content:"\e105"}.glyphicon-eye-close:before{content:"\e106"}.glyphicon-warning-sign:before{content:"\e107"}.glyphicon-plane:before{content:"\e108"}.glyphicon-calendar:before{content:"\e109"}.glyphicon-random:before{content:"\e110"}.glyphicon-comment:before{content:"\e111"}.glyphicon-magnet:before{content:"\e112"}.glyphicon-chevron-up:before{content:"\e113"}.glyphicon-chevron-down:before{content:"\e114"}.glyphicon-retweet:before{content:"\e115"}.glyphicon-shopping-cart:before{content:"\e116"}.glyphicon-folder-close:before{content:"\e117"}.glyphicon-folder-open:before{content:"\e118"}.glyphicon-resize-vertical:before{content:"\e119"}.glyphicon-resize-horizontal:before{content:"\e120"}.glyphicon-hdd:before{content:"\e121"}.glyphicon-bullhorn:before{content:"\e122"}.glyphicon-bell:before{content:"\e123"}.glyphicon-certificate:before{content:"\e124"}.glyphicon-thumbs-up:before{content:"\e125"}.glyphicon-thumbs-down:before{content:"\e126"}.glyphicon-hand-right:before{content:"\e127"}.glyphicon-hand-left:before{content:"\e128"}.glyphicon-hand-up:before{content:"\e129"}.glyphicon-hand-down:before{content:"\e130"}.glyphicon-circle-arrow-right:before{content:"\e131"}.glyphicon-circle-arrow-left:before{content:"\e132"}.glyphicon-circle-arrow-up:before{content:"\e133"}.glyphicon-circle-arrow-down:before{content:"\e134"}.glyphicon-globe:before{content:"\e135"}.glyphicon-wrench:before{content:"\e136"}.glyphicon-tasks:before{content:"\e137"}.glyphicon-filter:before{content:"\e138"}.glyphicon-briefcase:before{content:"\e139"}.glyphicon-fullscreen:before{content:"\e140"}.glyphicon-dashboard:before{content:"\e141"}.glyphicon-paperclip:before{content:"\e142"}.glyphicon-heart-empty:before{content:"\e143"}.glyphicon-link:before{content:"\e144"}.glyphicon-phone:before{content:"\e145"}.glyphicon-pushpin:before{content:"\e146"}.glyphicon-usd:before{content:"\e148"}.glyphicon-gbp:before{content:"\e149"}.glyphicon-sort:before{content:"\e150"}.glyphicon-sort-by-alphabet:before{content:"\e151"}.glyphicon-sort-by-alphabet-alt:before{content:"\e152"}.glyphicon-sort-by-order:before{content:"\e153"}.glyphicon-sort-by-order-alt:before{content:"\e154"}.glyphicon-sort-by-attributes:before{content:"\e155"}.glyphicon-sort-by-attributes-alt:before{content:"\e156"}.glyphicon-unchecked:before{content:"\e157"}.glyphicon-expand:before{content:"\e158"}.glyphicon-collapse-down:before{content:"\e159"}.glyphicon-collapse-up:before{content:"\e160"}.glyphicon-log-in:before{content:"\e161"}.glyphicon-flash:before{content:"\e162"}.glyphicon-log-out:before{content:"\e163"}.glyphicon-new-window:before{content:"\e164"}.glyphicon-record:before{content:"\e165"}.glyphicon-save:before{content:"\e166"}.glyphicon-open:before{content:"\e167"}.glyphicon-saved:before{content:"\e168"}.glyphicon-import:before{content:"\e169"}.glyphicon-export:before{content:"\e170"}.glyphicon-send:before{content:"\e171"}.glyphicon-floppy-disk:before{content:"\e172"}.glyphicon-floppy-saved:before{content:"\e173"}.glyphicon-floppy-remove:before{content:"\e174"}.glyphicon-floppy-save:before{content:"\e175"}.glyphicon-floppy-open:before{content:"\e176"}.glyphicon-credit-card:before{content:"\e177"}.glyphicon-transfer:before{content:"\e178"}.glyphicon-cutlery:before{content:"\e179"}.glyphicon-header:before{content:"\e180"}.glyphicon-compressed:before{content:"\e181"}.glyphicon-earphone:before{content:"\e182"}.glyphicon-phone-alt:before{content:"\e183"}.glyphicon-tower:before{content:"\e184"}.glyphicon-stats:before{content:"\e185"}.glyphicon-sd-video:before{content:"\e186"}.glyphicon-hd-video:before{content:"\e187"}.glyphicon-subtitles:before{content:"\e188"}.glyphicon-sound-stereo:before{content:"\e189"}.glyphicon-sound-dolby:before{content:"\e190"}.glyphicon-sound-5-1:before{content:"\e191"}.glyphicon-sound-6-1:before{content:"\e192"}.glyphicon-sound-7-1:before{content:"\e193"}.glyphicon-copyright-mark:before{content:"\e194"}.glyphicon-registration-mark:before{content:"\e195"}.glyphicon-cloud-download:before{content:"\e197"}.glyphicon-cloud-upload:before{content:"\e198"}.glyphicon-tree-conifer:before{content:"\e199"}.glyphicon-tree-deciduous:before{content:"\e200"}.caret{display:inline-block;width:0;height:0;margin-left:2px;vertical-align:middle;border-top:4px solid;border-right:4px solid transparent;border-left:4px solid transparent}.dropdown{position:relative}.dropdown-toggle:focus{outline:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;list-style:none;font-size:15px;background-color:#ffffff;border:1px solid #cccccc;border:1px solid rgba(0,0,0,0.15);border-radius:0;-webkit-box-shadow:0 6px 12px rgba(0,0,0,0.175);box-shadow:0 6px 12px rgba(0,0,0,0.175);background-clip:padding-box}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{height:1px;margin:9.5px 0;overflow:hidden;background-color:rgba(0,0,0,0.2)}.dropdown-menu>li>a{display:block;padding:3px 20px;clear:both;font-weight:normal;line-height:1.42857143;color:#555555;white-space:nowrap}.dropdown-menu>li>a:hover,.dropdown-menu>li>a:focus{text-decoration:none;color:#262626;background-color:#eeeeee}.dropdown-menu>.active>a,.dropdown-menu>.active>a:hover,.dropdown-menu>.active>a:focus{color:#ffffff;text-decoration:none;outline:0;background-color:#008cba}.dropdown-menu>.disabled>a,.dropdown-menu>.disabled>a:hover,.dropdown-menu>.disabled>a:focus{color:#999999}.dropdown-menu>.disabled>a:hover,.dropdown-menu>.disabled>a:focus{text-decoration:none;background-color:transparent;background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);cursor:not-allowed}.open>.dropdown-menu{display:block}.open>a{outline:0}.dropdown-menu-right{left:auto;right:0}.dropdown-menu-left{left:0;right:auto}.dropdown-header{display:block;padding:3px 20px;font-size:12px;line-height:1.42857143;color:#999999}.dropdown-backdrop{position:fixed;left:0;right:0;bottom:0;top:0;z-index:990}.pull-right>.dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{border-top:0;border-bottom:4px solid;content:""}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:1px}@media (min-width:768px){.navbar-right .dropdown-menu{left:auto;right:0}.navbar-right .dropdown-menu-left{left:0;right:auto}}.btn-group,.btn-group-vertical{position:relative;display:inline-block;vertical-align:middle}.btn-group>.btn,.btn-group-vertical>.btn{position:relative;float:left}.btn-group>.btn:hover,.btn-group-vertical>.btn:hover,.btn-group>.btn:focus,.btn-group-vertical>.btn:focus,.btn-group>.btn:active,.btn-group-vertical>.btn:active,.btn-group>.btn.active,.btn-group-vertical>.btn.active{z-index:2}.btn-group>.btn:focus,.btn-group-vertical>.btn:focus{outline:none}.btn-group .btn+.btn,.btn-group .btn+.btn-group,.btn-group .btn-group+.btn,.btn-group .btn-group+.btn-group{margin-left:-1px}.btn-toolbar{margin-left:-5px}.btn-toolbar .btn-group,.btn-toolbar .input-group{float:left}.btn-toolbar>.btn,.btn-toolbar>.btn-group,.btn-toolbar>.input-group{margin-left:5px}.btn-group>.btn:not(:first-child):not(:last-child):not(.dropdown-toggle){border-radius:0}.btn-group>.btn:first-child{margin-left:0}.btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-top-right-radius:0}.btn-group>.btn:last-child:not(:first-child),.btn-group>.dropdown-toggle:not(:first-child){border-bottom-left-radius:0;border-top-left-radius:0}.btn-group>.btn-group{float:left}.btn-group>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group>.btn-group:first-child>.btn:last-child,.btn-group>.btn-group:first-child>.dropdown-toggle{border-bottom-right-radius:0;border-top-right-radius:0}.btn-group>.btn-group:last-child>.btn:first-child{border-bottom-left-radius:0;border-top-left-radius:0}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn-group>.btn+.dropdown-toggle{padding-left:8px;padding-right:8px}.btn-group>.btn-lg+.dropdown-toggle{padding-left:12px;padding-right:12px}.btn-group.open .dropdown-toggle{-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,0.125);box-shadow:inset 0 3px 5px rgba(0,0,0,0.125)}.btn-group.open .dropdown-toggle.btn-link{-webkit-box-shadow:none;box-shadow:none}.btn .caret{margin-left:0}.btn-lg .caret{border-width:5px 5px 0;border-bottom-width:0}.dropup .btn-lg .caret{border-width:0 5px 5px}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group,.btn-group-vertical>.btn-group>.btn{display:block;float:none;width:100%;max-width:100%}.btn-group-vertical>.btn-group>.btn{float:none}.btn-group-vertical>.btn+.btn,.btn-group-vertical>.btn+.btn-group,.btn-group-vertical>.btn-group+.btn,.btn-group-vertical>.btn-group+.btn-group{margin-top:-1px;margin-left:0}.btn-group-vertical>.btn:not(:first-child):not(:last-child){border-radius:0}.btn-group-vertical>.btn:first-child:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:last-child:not(:first-child){border-bottom-left-radius:0;border-top-right-radius:0;border-top-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group-vertical>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group-vertical>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-right-radius:0;border-top-left-radius:0}.btn-group-justified{display:table;width:100%;table-layout:fixed;border-collapse:separate}.btn-group-justified>.btn,.btn-group-justified>.btn-group{float:none;display:table-cell;width:1%}.btn-group-justified>.btn-group .btn{width:100%}[data-toggle="buttons"]>.btn>input[type="radio"],[data-toggle="buttons"]>.btn>input[type="checkbox"]{display:none}.input-group{position:relative;display:table;border-collapse:separate}.input-group[class*="col-"]{float:none;padding-left:0;padding-right:0}.input-group .form-control{position:relative;z-index:2;float:left;width:100%;margin-bottom:0}.input-group-lg>.form-control,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.btn{height:48px;padding:10px 16px;font-size:19px;line-height:1.33;border-radius:0}select.input-group-lg>.form-control,select.input-group-lg>.input-group-addon,select.input-group-lg>.input-group-btn>.btn{height:48px;line-height:48px}textarea.input-group-lg>.form-control,textarea.input-group-lg>.input-group-addon,textarea.input-group-lg>.input-group-btn>.btn,select[multiple].input-group-lg>.form-control,select[multiple].input-group-lg>.input-group-addon,select[multiple].input-group-lg>.input-group-btn>.btn{height:auto}.input-group-sm>.form-control,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.btn{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:0}select.input-group-sm>.form-control,select.input-group-sm>.input-group-addon,select.input-group-sm>.input-group-btn>.btn{height:30px;line-height:30px}textarea.input-group-sm>.form-control,textarea.input-group-sm>.input-group-addon,textarea.input-group-sm>.input-group-btn>.btn,select[multiple].input-group-sm>.form-control,select[multiple].input-group-sm>.input-group-addon,select[multiple].input-group-sm>.input-group-btn>.btn{height:auto}.input-group-addon,.input-group-btn,.input-group .form-control{display:table-cell}.input-group-addon:not(:first-child):not(:last-child),.input-group-btn:not(:first-child):not(:last-child),.input-group .form-control:not(:first-child):not(:last-child){border-radius:0}.input-group-addon,.input-group-btn{width:1%;white-space:nowrap;vertical-align:middle}.input-group-addon{padding:6px 12px;font-size:15px;font-weight:normal;line-height:1;color:#6f6f6f;text-align:center;background-color:#eeeeee;border:1px solid #cccccc;border-radius:0}.input-group-addon.input-sm{padding:5px 10px;font-size:12px;border-radius:0}.input-group-addon.input-lg{padding:10px 16px;font-size:19px;border-radius:0}.input-group-addon input[type="radio"],.input-group-addon input[type="checkbox"]{margin-top:0}.input-group .form-control:first-child,.input-group-addon:first-child,.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group>.btn,.input-group-btn:first-child>.dropdown-toggle,.input-group-btn:last-child>.btn:not(:last-child):not(.dropdown-toggle),.input-group-btn:last-child>.btn-group:not(:last-child)>.btn{border-bottom-right-radius:0;border-top-right-radius:0}.input-group-addon:first-child{border-right:0}.input-group .form-control:last-child,.input-group-addon:last-child,.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group>.btn,.input-group-btn:last-child>.dropdown-toggle,.input-group-btn:first-child>.btn:not(:first-child),.input-group-btn:first-child>.btn-group:not(:first-child)>.btn{border-bottom-left-radius:0;border-top-left-radius:0}.input-group-addon:last-child{border-left:0}.input-group-btn{position:relative;font-size:0;white-space:nowrap}.input-group-btn>.btn{position:relative}.input-group-btn>.btn+.btn{margin-left:-1px}.input-group-btn>.btn:hover,.input-group-btn>.btn:focus,.input-group-btn>.btn:active{z-index:2}.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group{margin-right:-1px}.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group{margin-left:-1px}.nav{margin-bottom:0;padding-left:0;list-style:none}.nav>li{position:relative;display:block}.nav>li>a{position:relative;display:block;padding:10px 15px}.nav>li>a:hover,.nav>li>a:focus{text-decoration:none;background-color:#eeeeee}.nav>li.disabled>a{color:#999999}.nav>li.disabled>a:hover,.nav>li.disabled>a:focus{color:#999999;text-decoration:none;background-color:transparent;cursor:not-allowed}.nav .open>a,.nav .open>a:hover,.nav .open>a:focus{background-color:#eeeeee;border-color:#008cba}.nav .nav-divider{height:1px;margin:9.5px 0;overflow:hidden;background-color:#e5e5e5}.nav>li>a>img{max-width:none}.nav-tabs{border-bottom:1px solid #dddddd}.nav-tabs>li{float:left;margin-bottom:-1px}.nav-tabs>li>a{margin-right:2px;line-height:1.42857143;border:1px solid transparent;border-radius:0 0 0 0}.nav-tabs>li>a:hover{border-color:#eeeeee #eeeeee #dddddd}.nav-tabs>li.active>a,.nav-tabs>li.active>a:hover,.nav-tabs>li.active>a:focus{color:#6f6f6f;background-color:#ffffff;border:1px solid #dddddd;border-bottom-color:transparent;cursor:default}.nav-tabs.nav-justified{width:100%;border-bottom:0}.nav-tabs.nav-justified>li{float:none}.nav-tabs.nav-justified>li>a{text-align:center;margin-bottom:5px}.nav-tabs.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-tabs.nav-justified>li{display:table-cell;width:1%}.nav-tabs.nav-justified>li>a{margin-bottom:0}}.nav-tabs.nav-justified>li>a{margin-right:0;border-radius:0}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:hover,.nav-tabs.nav-justified>.active>a:focus{border:1px solid #dddddd}@media (min-width:768px){.nav-tabs.nav-justified>li>a{border-bottom:1px solid #dddddd;border-radius:0 0 0 0}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:hover,.nav-tabs.nav-justified>.active>a:focus{border-bottom-color:#ffffff}}.nav-pills>li{float:left}.nav-pills>li>a{border-radius:0}.nav-pills>li+li{margin-left:2px}.nav-pills>li.active>a,.nav-pills>li.active>a:hover,.nav-pills>li.active>a:focus{color:#ffffff;background-color:#008cba}.nav-stacked>li{float:none}.nav-stacked>li+li{margin-top:2px;margin-left:0}.nav-justified{width:100%}.nav-justified>li{float:none}.nav-justified>li>a{text-align:center;margin-bottom:5px}.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-justified>li{display:table-cell;width:1%}.nav-justified>li>a{margin-bottom:0}}.nav-tabs-justified{border-bottom:0}.nav-tabs-justified>li>a{margin-right:0;border-radius:0}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:hover,.nav-tabs-justified>.active>a:focus{border:1px solid #dddddd}@media (min-width:768px){.nav-tabs-justified>li>a{border-bottom:1px solid #dddddd;border-radius:0 0 0 0}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:hover,.nav-tabs-justified>.active>a:focus{border-bottom-color:#ffffff}}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-right-radius:0;border-top-left-radius:0}.navbar{position:relative;min-height:45px;margin-bottom:21px;border:1px solid transparent}@media (min-width:768px){.navbar{border-radius:0}}@media (min-width:768px){.navbar-header{float:left}}.navbar-collapse{max-height:340px;overflow-x:visible;padding-right:15px;padding-left:15px;border-top:1px solid transparent;box-shadow:inset 0 1px 0 rgba(255,255,255,0.1);-webkit-overflow-scrolling:touch}.navbar-collapse.in{overflow-y:auto}@media (min-width:768px){.navbar-collapse{width:auto;border-top:0;box-shadow:none}.navbar-collapse.collapse{display:block !important;height:auto !important;padding-bottom:0;overflow:visible !important}.navbar-collapse.in{overflow-y:visible}.navbar-fixed-top .navbar-collapse,.navbar-static-top .navbar-collapse,.navbar-fixed-bottom .navbar-collapse{padding-left:0;padding-right:0}}.container>.navbar-header,.container-fluid>.navbar-header,.container>.navbar-collapse,.container-fluid>.navbar-collapse{margin-right:-15px;margin-left:-15px}@media (min-width:768px){.container>.navbar-header,.container-fluid>.navbar-header,.container>.navbar-collapse,.container-fluid>.navbar-collapse{margin-right:0;margin-left:0}}.navbar-static-top{z-index:1000;border-width:0 0 1px}@media (min-width:768px){.navbar-static-top{border-radius:0}}.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;z-index:1030}@media (min-width:768px){.navbar-fixed-top,.navbar-fixed-bottom{border-radius:0}}.navbar-fixed-top{top:0;border-width:0 0 1px}.navbar-fixed-bottom{bottom:0;margin-bottom:0;border-width:1px 0 0}.navbar-brand{float:left;padding:12px 15px;font-size:19px;line-height:21px;height:45px}.navbar-brand:hover,.navbar-brand:focus{text-decoration:none}@media (min-width:768px){.navbar>.container .navbar-brand,.navbar>.container-fluid .navbar-brand{margin-left:-15px}}.navbar-toggle{position:relative;float:right;margin-right:15px;padding:9px 10px;margin-top:5.5px;margin-bottom:5.5px;background-color:transparent;background-image:none;border:1px solid transparent;border-radius:0}.navbar-toggle:focus{outline:none}.navbar-toggle .icon-bar{display:block;width:22px;height:2px;border-radius:1px}.navbar-toggle .icon-bar+.icon-bar{margin-top:4px}@media (min-width:768px){.navbar-toggle{display:none}}.navbar-nav{margin:6px -15px}.navbar-nav>li>a{padding-top:10px;padding-bottom:10px;line-height:21px}@media (max-width:767px){.navbar-nav .open .dropdown-menu{position:static;float:none;width:auto;margin-top:0;background-color:transparent;border:0;box-shadow:none}.navbar-nav .open .dropdown-menu>li>a,.navbar-nav .open .dropdown-menu .dropdown-header{padding:5px 15px 5px 25px}.navbar-nav .open .dropdown-menu>li>a{line-height:21px}.navbar-nav .open .dropdown-menu>li>a:hover,.navbar-nav .open .dropdown-menu>li>a:focus{background-image:none}}@media (min-width:768px){.navbar-nav{float:left;margin:0}.navbar-nav>li{float:left}.navbar-nav>li>a{padding-top:12px;padding-bottom:12px}.navbar-nav.navbar-right:last-child{margin-right:-15px}}@media (min-width:768px){.navbar-left{float:left !important}.navbar-right{float:right !important}}.navbar-form{margin-left:-15px;margin-right:-15px;padding:10px 15px;border-top:1px solid transparent;border-bottom:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);margin-top:5px;margin-bottom:5px}@media (min-width:768px){.navbar-form .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.navbar-form .form-control{display:inline-block;width:auto;vertical-align:middle}.navbar-form .input-group>.form-control{width:100%}.navbar-form .control-label{margin-bottom:0;vertical-align:middle}.navbar-form .radio,.navbar-form .checkbox{display:inline-block;margin-top:0;margin-bottom:0;padding-left:0;vertical-align:middle}.navbar-form .radio input[type="radio"],.navbar-form .checkbox input[type="checkbox"]{float:none;margin-left:0}.navbar-form .has-feedback .form-control-feedback{top:0}}@media (max-width:767px){.navbar-form .form-group{margin-bottom:5px}}@media (min-width:768px){.navbar-form{width:auto;border:0;margin-left:0;margin-right:0;padding-top:0;padding-bottom:0;-webkit-box-shadow:none;box-shadow:none}.navbar-form.navbar-right:last-child{margin-right:-15px}}.navbar-nav>li>.dropdown-menu{margin-top:0;border-top-right-radius:0;border-top-left-radius:0}.navbar-fixed-bottom .navbar-nav>li>.dropdown-menu{border-bottom-right-radius:0;border-bottom-left-radius:0}.navbar-btn{margin-top:5px;margin-bottom:5px}.navbar-btn.btn-sm{margin-top:7.5px;margin-bottom:7.5px}.navbar-btn.btn-xs{margin-top:11.5px;margin-bottom:11.5px}.navbar-text{margin-top:12px;margin-bottom:12px}@media (min-width:768px){.navbar-text{float:left;margin-left:15px;margin-right:15px}.navbar-text.navbar-right:last-child{margin-right:0}}.navbar-default{background-color:#333333;border-color:#222222}.navbar-default .navbar-brand{color:#ffffff}.navbar-default .navbar-brand:hover,.navbar-default .navbar-brand:focus{color:#ffffff;background-color:transparent}.navbar-default .navbar-text{color:#ffffff}.navbar-default .navbar-nav>li>a{color:#ffffff}.navbar-default .navbar-nav>li>a:hover,.navbar-default .navbar-nav>li>a:focus{color:#ffffff;background-color:#272727}.navbar-default .navbar-nav>.active>a,.navbar-default .navbar-nav>.active>a:hover,.navbar-default .navbar-nav>.active>a:focus{color:#ffffff;background-color:#272727}.navbar-default .navbar-nav>.disabled>a,.navbar-default .navbar-nav>.disabled>a:hover,.navbar-default .navbar-nav>.disabled>a:focus{color:#cccccc;background-color:transparent}.navbar-default .navbar-toggle{border-color:transparent}.navbar-default .navbar-toggle:hover,.navbar-default .navbar-toggle:focus{background-color:transparent}.navbar-default .navbar-toggle .icon-bar{background-color:#ffffff}.navbar-default .navbar-collapse,.navbar-default .navbar-form{border-color:#222222}.navbar-default .navbar-nav>.open>a,.navbar-default .navbar-nav>.open>a:hover,.navbar-default .navbar-nav>.open>a:focus{background-color:#272727;color:#ffffff}@media (max-width:767px){.navbar-default .navbar-nav .open .dropdown-menu>li>a{color:#ffffff}.navbar-default .navbar-nav .open .dropdown-menu>li>a:hover,.navbar-default .navbar-nav .open .dropdown-menu>li>a:focus{color:#ffffff;background-color:#272727}.navbar-default .navbar-nav .open .dropdown-menu>.active>a,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:hover,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:focus{color:#ffffff;background-color:#272727}.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:hover,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:focus{color:#cccccc;background-color:transparent}}.navbar-default .navbar-link{color:#ffffff}.navbar-default .navbar-link:hover{color:#ffffff}.navbar-inverse{background-color:#008cba;border-color:#006687}.navbar-inverse .navbar-brand{color:#ffffff}.navbar-inverse .navbar-brand:hover,.navbar-inverse .navbar-brand:focus{color:#ffffff;background-color:transparent}.navbar-inverse .navbar-text{color:#ffffff}.navbar-inverse .navbar-nav>li>a{color:#ffffff}.navbar-inverse .navbar-nav>li>a:hover,.navbar-inverse .navbar-nav>li>a:focus{color:#ffffff;background-color:#006687}.navbar-inverse .navbar-nav>.active>a,.navbar-inverse .navbar-nav>.active>a:hover,.navbar-inverse .navbar-nav>.active>a:focus{color:#ffffff;background-color:#006687}.navbar-inverse .navbar-nav>.disabled>a,.navbar-inverse .navbar-nav>.disabled>a:hover,.navbar-inverse .navbar-nav>.disabled>a:focus{color:#444444;background-color:transparent}.navbar-inverse .navbar-toggle{border-color:transparent}.navbar-inverse .navbar-toggle:hover,.navbar-inverse .navbar-toggle:focus{background-color:transparent}.navbar-inverse .navbar-toggle .icon-bar{background-color:#ffffff}.navbar-inverse .navbar-collapse,.navbar-inverse .navbar-form{border-color:#007196}.navbar-inverse .navbar-nav>.open>a,.navbar-inverse .navbar-nav>.open>a:hover,.navbar-inverse .navbar-nav>.open>a:focus{background-color:#006687;color:#ffffff}@media (max-width:767px){.navbar-inverse .navbar-nav .open .dropdown-menu>.dropdown-header{border-color:#006687}.navbar-inverse .navbar-nav .open .dropdown-menu .divider{background-color:#006687}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a{color:#ffffff}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:hover,.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:focus{color:#ffffff;background-color:#006687}.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:hover,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:focus{color:#ffffff;background-color:#006687}.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:hover,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:focus{color:#444444;background-color:transparent}}.navbar-inverse .navbar-link{color:#ffffff}.navbar-inverse .navbar-link:hover{color:#ffffff}.breadcrumb{padding:8px 15px;margin-bottom:21px;list-style:none;background-color:#f5f5f5;border-radius:0}.breadcrumb>li{display:inline-block}.breadcrumb>li+li:before{content:"/\00a0";padding:0 5px;color:#999999}.breadcrumb>.active{color:#333333}.pagination{display:inline-block;padding-left:0;margin:21px 0;border-radius:0}.pagination>li{display:inline}.pagination>li>a,.pagination>li>span{position:relative;float:left;padding:6px 12px;line-height:1.42857143;text-decoration:none;color:#008cba;background-color:transparent;border:1px solid transparent;margin-left:-1px}.pagination>li:first-child>a,.pagination>li:first-child>span{margin-left:0;border-bottom-left-radius:0;border-top-left-radius:0}.pagination>li:last-child>a,.pagination>li:last-child>span{border-bottom-right-radius:0;border-top-right-radius:0}.pagination>li>a:hover,.pagination>li>span:hover,.pagination>li>a:focus,.pagination>li>span:focus{color:#00526e;background-color:#eeeeee;border-color:transparent}.pagination>.active>a,.pagination>.active>span,.pagination>.active>a:hover,.pagination>.active>span:hover,.pagination>.active>a:focus,.pagination>.active>span:focus{z-index:2;color:#ffffff;background-color:#008cba;border-color:transparent;cursor:default}.pagination>.disabled>span,.pagination>.disabled>span:hover,.pagination>.disabled>span:focus,.pagination>.disabled>a,.pagination>.disabled>a:hover,.pagination>.disabled>a:focus{color:#999999;background-color:#ffffff;border-color:transparent;cursor:not-allowed}.pagination-lg>li>a,.pagination-lg>li>span{padding:10px 16px;font-size:19px}.pagination-lg>li:first-child>a,.pagination-lg>li:first-child>span{border-bottom-left-radius:0;border-top-left-radius:0}.pagination-lg>li:last-child>a,.pagination-lg>li:last-child>span{border-bottom-right-radius:0;border-top-right-radius:0}.pagination-sm>li>a,.pagination-sm>li>span{padding:5px 10px;font-size:12px}.pagination-sm>li:first-child>a,.pagination-sm>li:first-child>span{border-bottom-left-radius:0;border-top-left-radius:0}.pagination-sm>li:last-child>a,.pagination-sm>li:last-child>span{border-bottom-right-radius:0;border-top-right-radius:0}.pager{padding-left:0;margin:21px 0;list-style:none;text-align:center}.pager li{display:inline}.pager li>a,.pager li>span{display:inline-block;padding:5px 14px;background-color:transparent;border:1px solid transparent;border-radius:3px}.pager li>a:hover,.pager li>a:focus{text-decoration:none;background-color:#eeeeee}.pager .next>a,.pager .next>span{float:right}.pager .previous>a,.pager .previous>span{float:left}.pager .disabled>a,.pager .disabled>a:hover,.pager .disabled>a:focus,.pager .disabled>span{color:#999999;background-color:transparent;cursor:not-allowed}.label{display:inline;padding:.2em .6em .3em;font-size:75%;font-weight:bold;line-height:1;color:#ffffff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25em}.label[href]:hover,.label[href]:focus{color:#ffffff;text-decoration:none;cursor:pointer}.label:empty{display:none}.btn .label{position:relative;top:-1px}.label-default{background-color:#999999}.label-default[href]:hover,.label-default[href]:focus{background-color:#808080}.label-primary{background-color:#008cba}.label-primary[href]:hover,.label-primary[href]:focus{background-color:#006687}.label-success{background-color:#43ac6a}.label-success[href]:hover,.label-success[href]:focus{background-color:#358753}.label-info{background-color:#5bc0de}.label-info[href]:hover,.label-info[href]:focus{background-color:#31b0d5}.label-warning{background-color:#e99002}.label-warning[href]:hover,.label-warning[href]:focus{background-color:#b67102}.label-danger{background-color:#f04124}.label-danger[href]:hover,.label-danger[href]:focus{background-color:#d32a0e}.badge{display:inline-block;min-width:10px;padding:3px 7px;font-size:12px;font-weight:bold;color:#777777;line-height:1;vertical-align:baseline;white-space:nowrap;text-align:center;background-color:#e7e7e7;border-radius:10px}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.btn-xs .badge{top:0;padding:1px 5px}a.badge:hover,a.badge:focus{color:#ffffff;text-decoration:none;cursor:pointer}a.list-group-item.active>.badge,.nav-pills>.active>a>.badge{color:#008cba;background-color:#ffffff}.nav-pills>li>a>.badge{margin-left:3px}.jumbotron{padding:30px;margin-bottom:30px;color:inherit;background-color:#fafafa}.jumbotron h1,.jumbotron .h1{color:inherit}.jumbotron p{margin-bottom:15px;font-size:23px;font-weight:200}.container .jumbotron{border-radius:0}.jumbotron .container{max-width:100%}@media screen and (min-width:768px){.jumbotron{padding-top:48px;padding-bottom:48px}.container .jumbotron{padding-left:60px;padding-right:60px}.jumbotron h1,.jumbotron .h1{font-size:67.5px}}.thumbnail{display:block;padding:4px;margin-bottom:21px;line-height:1.42857143;background-color:#ffffff;border:1px solid #dddddd;border-radius:0;-webkit-transition:all .2s ease-in-out;transition:all .2s ease-in-out}.thumbnail>img,.thumbnail a>img{margin-left:auto;margin-right:auto}a.thumbnail:hover,a.thumbnail:focus,a.thumbnail.active{border-color:#008cba}.thumbnail .caption{padding:9px;color:#222222}.alert{padding:15px;margin-bottom:21px;border:1px solid transparent;border-radius:0}.alert h4{margin-top:0;color:inherit}.alert .alert-link{font-weight:bold}.alert>p,.alert>ul{margin-bottom:0}.alert>p+p{margin-top:5px}.alert-dismissable{padding-right:35px}.alert-dismissable .close{position:relative;top:-2px;right:-21px;color:inherit}.alert-success{background-color:#43ac6a;border-color:#3c9a5f;color:#ffffff}.alert-success hr{border-top-color:#358753}.alert-success .alert-link{color:#e6e6e6}.alert-info{background-color:#5bc0de;border-color:#3db5d8;color:#ffffff}.alert-info hr{border-top-color:#2aabd2}.alert-info .alert-link{color:#e6e6e6}.alert-warning{background-color:#e99002;border-color:#d08002;color:#ffffff}.alert-warning hr{border-top-color:#b67102}.alert-warning .alert-link{color:#e6e6e6}.alert-danger{background-color:#f04124;border-color:#ea2f10;color:#ffffff}.alert-danger hr{border-top-color:#d32a0e}.alert-danger .alert-link{color:#e6e6e6}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{overflow:hidden;height:21px;margin-bottom:21px;background-color:#f5f5f5;border-radius:0;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1)}.progress-bar{float:left;width:0%;height:100%;font-size:12px;line-height:21px;color:#ffffff;text-align:center;background-color:#008cba;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-webkit-transition:width 0.6s ease;transition:width 0.6s ease}.progress-striped .progress-bar{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-size:40px 40px}.progress.active .progress-bar{-webkit-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-bar-success{background-color:#43ac6a}.progress-striped .progress-bar-success{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.progress-bar-info{background-color:#5bc0de}.progress-striped .progress-bar-info{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.progress-bar-warning{background-color:#e99002}.progress-striped .progress-bar-warning{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.progress-bar-danger{background-color:#f04124}.progress-striped .progress-bar-danger{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.media,.media-body{overflow:hidden;zoom:1}.media,.media .media{margin-top:15px}.media:first-child{margin-top:0}.media-object{display:block}.media-heading{margin:0 0 5px}.media>.pull-left{margin-right:10px}.media>.pull-right{margin-left:10px}.media-list{padding-left:0;list-style:none}.list-group{margin-bottom:20px;padding-left:0}.list-group-item{position:relative;display:block;padding:10px 15px;margin-bottom:-1px;background-color:#ffffff;border:1px solid #dddddd}.list-group-item:first-child{border-top-right-radius:0;border-top-left-radius:0}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:0;border-bottom-left-radius:0}.list-group-item>.badge{float:right}.list-group-item>.badge+.badge{margin-right:5px}a.list-group-item{color:#555555}a.list-group-item .list-group-item-heading{color:#333333}a.list-group-item:hover,a.list-group-item:focus{text-decoration:none;background-color:#f5f5f5}a.list-group-item.active,a.list-group-item.active:hover,a.list-group-item.active:focus{z-index:2;color:#ffffff;background-color:#008cba;border-color:#008cba}a.list-group-item.active .list-group-item-heading,a.list-group-item.active:hover .list-group-item-heading,a.list-group-item.active:focus .list-group-item-heading{color:inherit}a.list-group-item.active .list-group-item-text,a.list-group-item.active:hover .list-group-item-text,a.list-group-item.active:focus .list-group-item-text{color:#87e1ff}.list-group-item-success{color:#43ac6a;background-color:#dff0d8}a.list-group-item-success{color:#43ac6a}a.list-group-item-success .list-group-item-heading{color:inherit}a.list-group-item-success:hover,a.list-group-item-success:focus{color:#43ac6a;background-color:#d0e9c6}a.list-group-item-success.active,a.list-group-item-success.active:hover,a.list-group-item-success.active:focus{color:#fff;background-color:#43ac6a;border-color:#43ac6a}.list-group-item-info{color:#5bc0de;background-color:#d9edf7}a.list-group-item-info{color:#5bc0de}a.list-group-item-info .list-group-item-heading{color:inherit}a.list-group-item-info:hover,a.list-group-item-info:focus{color:#5bc0de;background-color:#c4e3f3}a.list-group-item-info.active,a.list-group-item-info.active:hover,a.list-group-item-info.active:focus{color:#fff;background-color:#5bc0de;border-color:#5bc0de}.list-group-item-warning{color:#e99002;background-color:#fcf8e3}a.list-group-item-warning{color:#e99002}a.list-group-item-warning .list-group-item-heading{color:inherit}a.list-group-item-warning:hover,a.list-group-item-warning:focus{color:#e99002;background-color:#faf2cc}a.list-group-item-warning.active,a.list-group-item-warning.active:hover,a.list-group-item-warning.active:focus{color:#fff;background-color:#e99002;border-color:#e99002}.list-group-item-danger{color:#f04124;background-color:#f2dede}a.list-group-item-danger{color:#f04124}a.list-group-item-danger .list-group-item-heading{color:inherit}a.list-group-item-danger:hover,a.list-group-item-danger:focus{color:#f04124;background-color:#ebcccc}a.list-group-item-danger.active,a.list-group-item-danger.active:hover,a.list-group-item-danger.active:focus{color:#fff;background-color:#f04124;border-color:#f04124}.list-group-item-heading{margin-top:0;margin-bottom:5px}.list-group-item-text{margin-bottom:0;line-height:1.3}.panel{margin-bottom:21px;background-color:#ffffff;border:1px solid transparent;border-radius:0;-webkit-box-shadow:0 1px 1px rgba(0,0,0,0.05);box-shadow:0 1px 1px rgba(0,0,0,0.05)}.panel-body{padding:15px}.panel-heading{padding:10px 15px;border-bottom:1px solid transparent;border-top-right-radius:-1;border-top-left-radius:-1}.panel-heading>.dropdown .dropdown-toggle{color:inherit}.panel-title{margin-top:0;margin-bottom:0;font-size:17px;color:inherit}.panel-title>a{color:inherit}.panel-footer{padding:10px 15px;background-color:#f5f5f5;border-top:1px solid #dddddd;border-bottom-right-radius:-1;border-bottom-left-radius:-1}.panel>.list-group{margin-bottom:0}.panel>.list-group .list-group-item{border-width:1px 0;border-radius:0}.panel>.list-group:first-child .list-group-item:first-child{border-top:0;border-top-right-radius:-1;border-top-left-radius:-1}.panel>.list-group:last-child .list-group-item:last-child{border-bottom:0;border-bottom-right-radius:-1;border-bottom-left-radius:-1}.panel-heading+.list-group .list-group-item:first-child{border-top-width:0}.panel>.table,.panel>.table-responsive>.table{margin-bottom:0}.panel>.table:first-child,.panel>.table-responsive:first-child>.table:first-child{border-top-right-radius:-1;border-top-left-radius:-1}.panel>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table:first-child>thead:first-child>tr:first-child th:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:first-child{border-top-left-radius:-1}.panel>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table:first-child>thead:first-child>tr:first-child th:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:last-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:last-child{border-top-right-radius:-1}.panel>.table:last-child,.panel>.table-responsive:last-child>.table:last-child{border-bottom-right-radius:-1;border-bottom-left-radius:-1}.panel>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:first-child{border-bottom-left-radius:-1}.panel>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:last-child{border-bottom-right-radius:-1}.panel>.panel-body+.table,.panel>.panel-body+.table-responsive{border-top:1px solid #dddddd}.panel>.table>tbody:first-child>tr:first-child th,.panel>.table>tbody:first-child>tr:first-child td{border-top:0}.panel>.table-bordered,.panel>.table-responsive>.table-bordered{border:0}.panel>.table-bordered>thead>tr>th:first-child,.panel>.table-responsive>.table-bordered>thead>tr>th:first-child,.panel>.table-bordered>tbody>tr>th:first-child,.panel>.table-responsive>.table-bordered>tbody>tr>th:first-child,.panel>.table-bordered>tfoot>tr>th:first-child,.panel>.table-responsive>.table-bordered>tfoot>tr>th:first-child,.panel>.table-bordered>thead>tr>td:first-child,.panel>.table-responsive>.table-bordered>thead>tr>td:first-child,.panel>.table-bordered>tbody>tr>td:first-child,.panel>.table-responsive>.table-bordered>tbody>tr>td:first-child,.panel>.table-bordered>tfoot>tr>td:first-child,.panel>.table-responsive>.table-bordered>tfoot>tr>td:first-child{border-left:0}.panel>.table-bordered>thead>tr>th:last-child,.panel>.table-responsive>.table-bordered>thead>tr>th:last-child,.panel>.table-bordered>tbody>tr>th:last-child,.panel>.table-responsive>.table-bordered>tbody>tr>th:last-child,.panel>.table-bordered>tfoot>tr>th:last-child,.panel>.table-responsive>.table-bordered>tfoot>tr>th:last-child,.panel>.table-bordered>thead>tr>td:last-child,.panel>.table-responsive>.table-bordered>thead>tr>td:last-child,.panel>.table-bordered>tbody>tr>td:last-child,.panel>.table-responsive>.table-bordered>tbody>tr>td:last-child,.panel>.table-bordered>tfoot>tr>td:last-child,.panel>.table-responsive>.table-bordered>tfoot>tr>td:last-child{border-right:0}.panel>.table-bordered>thead>tr:first-child>td,.panel>.table-responsive>.table-bordered>thead>tr:first-child>td,.panel>.table-bordered>tbody>tr:first-child>td,.panel>.table-responsive>.table-bordered>tbody>tr:first-child>td,.panel>.table-bordered>thead>tr:first-child>th,.panel>.table-responsive>.table-bordered>thead>tr:first-child>th,.panel>.table-bordered>tbody>tr:first-child>th,.panel>.table-responsive>.table-bordered>tbody>tr:first-child>th{border-bottom:0}.panel>.table-bordered>tbody>tr:last-child>td,.panel>.table-responsive>.table-bordered>tbody>tr:last-child>td,.panel>.table-bordered>tfoot>tr:last-child>td,.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>td,.panel>.table-bordered>tbody>tr:last-child>th,.panel>.table-responsive>.table-bordered>tbody>tr:last-child>th,.panel>.table-bordered>tfoot>tr:last-child>th,.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>th{border-bottom:0}.panel>.table-responsive{border:0;margin-bottom:0}.panel-group{margin-bottom:21px}.panel-group .panel{margin-bottom:0;border-radius:0;overflow:hidden}.panel-group .panel+.panel{margin-top:5px}.panel-group .panel-heading{border-bottom:0}.panel-group .panel-heading+.panel-collapse .panel-body{border-top:1px solid #dddddd}.panel-group .panel-footer{border-top:0}.panel-group .panel-footer+.panel-collapse .panel-body{border-bottom:1px solid #dddddd}.panel-default{border-color:#dddddd}.panel-default>.panel-heading{color:#333333;background-color:#f5f5f5;border-color:#dddddd}.panel-default>.panel-heading+.panel-collapse .panel-body{border-top-color:#dddddd}.panel-default>.panel-footer+.panel-collapse .panel-body{border-bottom-color:#dddddd}.panel-primary{border-color:#008cba}.panel-primary>.panel-heading{color:#ffffff;background-color:#008cba;border-color:#008cba}.panel-primary>.panel-heading+.panel-collapse .panel-body{border-top-color:#008cba}.panel-primary>.panel-footer+.panel-collapse .panel-body{border-bottom-color:#008cba}.panel-success{border-color:#3c9a5f}.panel-success>.panel-heading{color:#43ac6a;background-color:#dff0d8;border-color:#3c9a5f}.panel-success>.panel-heading+.panel-collapse .panel-body{border-top-color:#3c9a5f}.panel-success>.panel-footer+.panel-collapse .panel-body{border-bottom-color:#3c9a5f}.panel-info{border-color:#3db5d8}.panel-info>.panel-heading{color:#5bc0de;background-color:#d9edf7;border-color:#3db5d8}.panel-info>.panel-heading+.panel-collapse .panel-body{border-top-color:#3db5d8}.panel-info>.panel-footer+.panel-collapse .panel-body{border-bottom-color:#3db5d8}.panel-warning{border-color:#d08002}.panel-warning>.panel-heading{color:#e99002;background-color:#fcf8e3;border-color:#d08002}.panel-warning>.panel-heading+.panel-collapse .panel-body{border-top-color:#d08002}.panel-warning>.panel-footer+.panel-collapse .panel-body{border-bottom-color:#d08002}.panel-danger{border-color:#ea2f10}.panel-danger>.panel-heading{color:#f04124;background-color:#f2dede;border-color:#ea2f10}.panel-danger>.panel-heading+.panel-collapse .panel-body{border-top-color:#ea2f10}.panel-danger>.panel-footer+.panel-collapse .panel-body{border-bottom-color:#ea2f10}.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#fafafa;border:1px solid #e8e8e8;border-radius:0;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);box-shadow:inset 0 1px 1px rgba(0,0,0,0.05)}.well blockquote{border-color:#ddd;border-color:rgba(0,0,0,0.15)}.well-lg{padding:24px;border-radius:0}.well-sm{padding:9px;border-radius:0}.close{float:right;font-size:22.5px;font-weight:bold;line-height:1;color:#000000;text-shadow:0 1px 0 #ffffff;opacity:0.2;filter:alpha(opacity=20)}.close:hover,.close:focus{color:#000000;text-decoration:none;cursor:pointer;opacity:0.5;filter:alpha(opacity=50)}button.close{padding:0;cursor:pointer;background:transparent;border:0;-webkit-appearance:none}.modal-open{overflow:hidden}.modal{display:none;overflow:auto;overflow-y:scroll;position:fixed;top:0;right:0;bottom:0;left:0;z-index:1050;-webkit-overflow-scrolling:touch;outline:0}.modal.fade .modal-dialog{-webkit-transform:translate(0, -25%);-ms-transform:translate(0, -25%);transform:translate(0, -25%);-webkit-transition:-webkit-transform .3s ease-out;-moz-transition:-moz-transform .3s ease-out;-o-transition:-o-transform .3s ease-out;transition:transform .3s ease-out}.modal.in .modal-dialog{-webkit-transform:translate(0, 0);-ms-transform:translate(0, 0);transform:translate(0, 0)}.modal-dialog{position:relative;width:auto;margin:10px}.modal-content{position:relative;background-color:#ffffff;border:1px solid #999999;border:1px solid rgba(0,0,0,0.2);border-radius:0;-webkit-box-shadow:0 3px 9px rgba(0,0,0,0.5);box-shadow:0 3px 9px rgba(0,0,0,0.5);background-clip:padding-box;outline:none}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000000}.modal-backdrop.fade{opacity:0;filter:alpha(opacity=0)}.modal-backdrop.in{opacity:0.5;filter:alpha(opacity=50)}.modal-header{padding:15px;border-bottom:1px solid #e5e5e5;min-height:16.42857143px}.modal-header .close{margin-top:-2px}.modal-title{margin:0;line-height:1.42857143}.modal-body{position:relative;padding:20px}.modal-footer{margin-top:15px;padding:19px 20px 20px;text-align:right;border-top:1px solid #e5e5e5}.modal-footer .btn+.btn{margin-left:5px;margin-bottom:0}.modal-footer .btn-group .btn+.btn{margin-left:-1px}.modal-footer .btn-block+.btn-block{margin-left:0}@media (min-width:768px){.modal-dialog{width:600px;margin:30px auto}.modal-content{-webkit-box-shadow:0 5px 15px rgba(0,0,0,0.5);box-shadow:0 5px 15px rgba(0,0,0,0.5)}.modal-sm{width:300px}}@media (min-width:992px){.modal-lg{width:900px}}.tooltip{position:absolute;z-index:1030;display:block;visibility:visible;font-size:12px;line-height:1.4;opacity:0;filter:alpha(opacity=0)}.tooltip.in{opacity:0.9;filter:alpha(opacity=90)}.tooltip.top{margin-top:-3px;padding:5px 0}.tooltip.right{margin-left:3px;padding:0 5px}.tooltip.bottom{margin-top:3px;padding:5px 0}.tooltip.left{margin-left:-3px;padding:0 5px}.tooltip-inner{max-width:200px;padding:3px 8px;color:#ffffff;text-align:center;text-decoration:none;background-color:#333333;border-radius:0}.tooltip-arrow{position:absolute;width:0;height:0;border-color:transparent;border-style:solid}.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-width:5px 5px 0;border-top-color:#333333}.tooltip.top-left .tooltip-arrow{bottom:0;left:5px;border-width:5px 5px 0;border-top-color:#333333}.tooltip.top-right .tooltip-arrow{bottom:0;right:5px;border-width:5px 5px 0;border-top-color:#333333}.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-width:5px 5px 5px 0;border-right-color:#333333}.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-width:5px 0 5px 5px;border-left-color:#333333}.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-width:0 5px 5px;border-bottom-color:#333333}.tooltip.bottom-left .tooltip-arrow{top:0;left:5px;border-width:0 5px 5px;border-bottom-color:#333333}.tooltip.bottom-right .tooltip-arrow{top:0;right:5px;border-width:0 5px 5px;border-bottom-color:#333333}.popover{position:absolute;top:0;left:0;z-index:1010;display:none;max-width:276px;padding:1px;text-align:left;background-color:#333333;background-clip:padding-box;border:1px solid #333333;border:1px solid transparent;border-radius:0;-webkit-box-shadow:0 5px 10px rgba(0,0,0,0.2);box-shadow:0 5px 10px rgba(0,0,0,0.2);white-space:normal}.popover.top{margin-top:-10px}.popover.right{margin-left:10px}.popover.bottom{margin-top:10px}.popover.left{margin-left:-10px}.popover-title{margin:0;padding:8px 14px;font-size:15px;font-weight:normal;line-height:18px;background-color:#333333;border-bottom:1px solid #262626;border-radius:5px 5px 0 0}.popover-content{padding:9px 14px}.popover>.arrow,.popover>.arrow:after{position:absolute;display:block;width:0;height:0;border-color:transparent;border-style:solid}.popover>.arrow{border-width:11px}.popover>.arrow:after{border-width:10px;content:""}.popover.top>.arrow{left:50%;margin-left:-11px;border-bottom-width:0;border-top-color:#000000;border-top-color:rgba(0,0,0,0.05);bottom:-11px}.popover.top>.arrow:after{content:" ";bottom:1px;margin-left:-10px;border-bottom-width:0;border-top-color:#333333}.popover.right>.arrow{top:50%;left:-11px;margin-top:-11px;border-left-width:0;border-right-color:#000000;border-right-color:rgba(0,0,0,0.05)}.popover.right>.arrow:after{content:" ";left:1px;bottom:-10px;border-left-width:0;border-right-color:#333333}.popover.bottom>.arrow{left:50%;margin-left:-11px;border-top-width:0;border-bottom-color:#000000;border-bottom-color:rgba(0,0,0,0.05);top:-11px}.popover.bottom>.arrow:after{content:" ";top:1px;margin-left:-10px;border-top-width:0;border-bottom-color:#333333}.popover.left>.arrow{top:50%;right:-11px;margin-top:-11px;border-right-width:0;border-left-color:#000000;border-left-color:rgba(0,0,0,0.05)}.popover.left>.arrow:after{content:" ";right:1px;border-right-width:0;border-left-color:#333333;bottom:-10px}.carousel{position:relative}.carousel-inner{position:relative;overflow:hidden;width:100%}.carousel-inner>.item{display:none;position:relative;-webkit-transition:.6s ease-in-out left;transition:.6s ease-in-out left}.carousel-inner>.item>img,.carousel-inner>.item>a>img{line-height:1}.carousel-inner>.active,.carousel-inner>.next,.carousel-inner>.prev{display:block}.carousel-inner>.active{left:0}.carousel-inner>.next,.carousel-inner>.prev{position:absolute;top:0;width:100%}.carousel-inner>.next{left:100%}.carousel-inner>.prev{left:-100%}.carousel-inner>.next.left,.carousel-inner>.prev.right{left:0}.carousel-inner>.active.left{left:-100%}.carousel-inner>.active.right{left:100%}.carousel-control{position:absolute;top:0;left:0;bottom:0;width:15%;opacity:0.5;filter:alpha(opacity=50);font-size:20px;color:#ffffff;text-align:center;text-shadow:0 1px 2px rgba(0,0,0,0.6)}.carousel-control.left{background-image:-webkit-linear-gradient(left, color-stop(rgba(0,0,0,0.5) 0), color-stop(rgba(0,0,0,0.0001) 100%));background-image:linear-gradient(to right, rgba(0,0,0,0.5) 0, rgba(0,0,0,0.0001) 100%);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#80000000', endColorstr='#00000000', GradientType=1)}.carousel-control.right{left:auto;right:0;background-image:-webkit-linear-gradient(left, color-stop(rgba(0,0,0,0.0001) 0), color-stop(rgba(0,0,0,0.5) 100%));background-image:linear-gradient(to right, rgba(0,0,0,0.0001) 0, rgba(0,0,0,0.5) 100%);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#00000000', endColorstr='#80000000', GradientType=1)}.carousel-control:hover,.carousel-control:focus{outline:none;color:#ffffff;text-decoration:none;opacity:0.9;filter:alpha(opacity=90)}.carousel-control .icon-prev,.carousel-control .icon-next,.carousel-control .glyphicon-chevron-left,.carousel-control .glyphicon-chevron-right{position:absolute;top:50%;z-index:5;display:inline-block}.carousel-control .icon-prev,.carousel-control .glyphicon-chevron-left{left:50%}.carousel-control .icon-next,.carousel-control .glyphicon-chevron-right{right:50%}.carousel-control .icon-prev,.carousel-control .icon-next{width:20px;height:20px;margin-top:-10px;margin-left:-10px;font-family:serif}.carousel-control .icon-prev:before{content:'\2039'}.carousel-control .icon-next:before{content:'\203a'}.carousel-indicators{position:absolute;bottom:10px;left:50%;z-index:15;width:60%;margin-left:-30%;padding-left:0;list-style:none;text-align:center}.carousel-indicators li{display:inline-block;width:10px;height:10px;margin:1px;text-indent:-999px;border:1px solid #ffffff;border-radius:10px;cursor:pointer;background-color:#000 \9;background-color:rgba(0,0,0,0)}.carousel-indicators .active{margin:0;width:12px;height:12px;background-color:#ffffff}.carousel-caption{position:absolute;left:15%;right:15%;bottom:20px;z-index:10;padding-top:20px;padding-bottom:20px;color:#ffffff;text-align:center;text-shadow:0 1px 2px rgba(0,0,0,0.6)}.carousel-caption .btn{text-shadow:none}@media screen and (min-width:768px){.carousel-control .glyphicon-chevron-left,.carousel-control .glyphicon-chevron-right,.carousel-control .icon-prev,.carousel-control .icon-next{width:30px;height:30px;margin-top:-15px;margin-left:-15px;font-size:30px}.carousel-caption{left:20%;right:20%;padding-bottom:30px}.carousel-indicators{bottom:20px}}.clearfix:before,.clearfix:after,.container:before,.container:after,.container-fluid:before,.container-fluid:after,.row:before,.row:after,.form-horizontal .form-group:before,.form-horizontal .form-group:after,.btn-toolbar:before,.btn-toolbar:after,.btn-group-vertical>.btn-group:before,.btn-group-vertical>.btn-group:after,.nav:before,.nav:after,.navbar:before,.navbar:after,.navbar-header:before,.navbar-header:after,.navbar-collapse:before,.navbar-collapse:after,.pager:before,.pager:after,.panel-body:before,.panel-body:after,.modal-footer:before,.modal-footer:after{content:" ";display:table}.clearfix:after,.container:after,.container-fluid:after,.row:after,.form-horizontal .form-group:after,.btn-toolbar:after,.btn-group-vertical>.btn-group:after,.nav:after,.navbar:after,.navbar-header:after,.navbar-collapse:after,.pager:after,.panel-body:after,.modal-footer:after{clear:both}.center-block{display:block;margin-left:auto;margin-right:auto}.pull-right{float:right !important}.pull-left{float:left !important}.hide{display:none !important}.show{display:block !important}.invisible{visibility:hidden}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.hidden{display:none !important;visibility:hidden !important}.affix{position:fixed}@-ms-viewport{width:device-width}.visible-xs,.visible-sm,.visible-md,.visible-lg{display:none !important}@media (max-width:767px){.visible-xs{display:block !important}table.visible-xs{display:table}tr.visible-xs{display:table-row !important}th.visible-xs,td.visible-xs{display:table-cell !important}}@media (min-width:768px) and (max-width:991px){.visible-sm{display:block !important}table.visible-sm{display:table}tr.visible-sm{display:table-row !important}th.visible-sm,td.visible-sm{display:table-cell !important}}@media (min-width:992px) and (max-width:1199px){.visible-md{display:block !important}table.visible-md{display:table}tr.visible-md{display:table-row !important}th.visible-md,td.visible-md{display:table-cell !important}}@media (min-width:1200px){.visible-lg{display:block !important}table.visible-lg{display:table}tr.visible-lg{display:table-row !important}th.visible-lg,td.visible-lg{display:table-cell !important}}@media (max-width:767px){.hidden-xs{display:none !important}}@media (min-width:768px) and (max-width:991px){.hidden-sm{display:none !important}}@media (min-width:992px) and (max-width:1199px){.hidden-md{display:none !important}}@media (min-width:1200px){.hidden-lg{display:none !important}}.visible-print{display:none !important}@media print{.visible-print{display:block !important}table.visible-print{display:table}tr.visible-print{display:table-row !important}th.visible-print,td.visible-print{display:table-cell !important}}@media print{.hidden-print{display:none !important}}.navbar{border:none;font-size:13px;font-weight:300}.navbar .navbar-toggle:hover .icon-bar{background-color:#b3b3b3}.navbar-collapse{border-top-color:rgba(0,0,0,0.2);-webkit-box-shadow:none;box-shadow:none}.navbar .btn{padding-top:6px;padding-bottom:6px}.navbar .dropdown-menu{border:none}.navbar .dropdown-menu>li>a,.navbar .dropdown-menu>li>a:focus{background-color:transparent;font-size:13px;font-weight:300}.navbar .dropdown-header{color:rgba(255,255,255,0.5)}.navbar-default .dropdown-menu{background-color:#333333}.navbar-default .dropdown-menu>li>a,.navbar-default .dropdown-menu>li>a:focus{color:#ffffff}.navbar-default .dropdown-menu>li>a:hover,.navbar-default .dropdown-menu>.active>a,.navbar-default .dropdown-menu>.active>a:hover{background-color:#272727}.navbar-inverse .dropdown-menu{background-color:#008cba}.navbar-inverse .dropdown-menu>li>a,.navbar-inverse .dropdown-menu>li>a:focus{color:#ffffff}.navbar-inverse .dropdown-menu>li>a:hover,.navbar-inverse .dropdown-menu>.active>a,.navbar-inverse .dropdown-menu>.active>a:hover{background-color:#006687}.btn{padding:14px 28px}.btn-lg{padding:16px 32px}.btn-sm{padding:8px 16px}.btn-xs{padding:4px 8px}.btn-group .btn~.dropdown-toggle{padding-left:16px;padding-right:16px}.btn-group .dropdown-menu{border-top-width:0}.btn-group.dropup .dropdown-menu{border-top-width:1px;border-bottom-width:0;margin-bottom:0}.btn-group .dropdown-toggle.btn-default~.dropdown-menu{background-color:#e7e7e7;border-color:#dadada}.btn-group .dropdown-toggle.btn-default~.dropdown-menu>li>a{color:#333333}.btn-group .dropdown-toggle.btn-default~.dropdown-menu>li>a:hover{background-color:#d3d3d3}.btn-group .dropdown-toggle.btn-primary~.dropdown-menu{background-color:#008cba;border-color:#0079a1}.btn-group .dropdown-toggle.btn-primary~.dropdown-menu>li>a{color:#ffffff}.btn-group .dropdown-toggle.btn-primary~.dropdown-menu>li>a:hover{background-color:#006d91}.btn-group .dropdown-toggle.btn-success~.dropdown-menu{background-color:#43ac6a;border-color:#3c9a5f}.btn-group .dropdown-toggle.btn-success~.dropdown-menu>li>a{color:#ffffff}.btn-group .dropdown-toggle.btn-success~.dropdown-menu>li>a:hover{background-color:#388f58}.btn-group .dropdown-toggle.btn-info~.dropdown-menu{background-color:#5bc0de;border-color:#46b8da}.btn-group .dropdown-toggle.btn-info~.dropdown-menu>li>a{color:#ffffff}.btn-group .dropdown-toggle.btn-info~.dropdown-menu>li>a:hover{background-color:#39b3d7}.btn-group .dropdown-toggle.btn-warning~.dropdown-menu{background-color:#e99002;border-color:#d08002}.btn-group .dropdown-toggle.btn-warning~.dropdown-menu>li>a{color:#ffffff}.btn-group .dropdown-toggle.btn-warning~.dropdown-menu>li>a:hover{background-color:#c17702}.btn-group .dropdown-toggle.btn-danger~.dropdown-menu{background-color:#f04124;border-color:#ea2f10}.btn-group .dropdown-toggle.btn-danger~.dropdown-menu>li>a{color:#ffffff}.btn-group .dropdown-toggle.btn-danger~.dropdown-menu>li>a:hover{background-color:#dc2c0f}.lead{color:#6f6f6f}cite{font-style:italic}blockquote{border-left-width:1px;color:#6f6f6f}blockquote.pull-right{border-right-width:1px}blockquote small{font-size:12px;font-weight:300}table{font-size:12px}input,.form-control{padding:7px}label,.control-label,.help-block,.checkbox,.radio{font-size:12px;font-weight:normal}.input-group-addon,.input-group-btn .btn{padding:8px 14px;font-size:12px}.nav .open>a,.nav .open>a:hover,.nav .open>a:focus{border-color:transparent}.nav-tabs>li>a{background-color:#e7e7e7;color:#222222}.nav-tabs .caret{border-top-color:#222222;border-bottom-color:#222222}.nav-pills{font-weight:300}.breadcrumb{border:1px solid #dddddd;border-radius:3px;font-size:10px;font-weight:300;text-transform:uppercase}.pagination{font-size:12px;font-weight:300;color:#999999}.pagination>li>a,.pagination>li>span{margin-left:4px;color:#999999}.pagination>.active>a,.pagination>.active>span{color:#fff}.pagination>li>a,.pagination>li:first-child>a,.pagination>li:last-child>a,.pagination>li>span,.pagination>li:first-child>span,.pagination>li:last-child>span{border-radius:3px}.pagination-lg>li>a{padding-left:22px;padding-right:22px}.pagination-sm>li>a{padding:0 5px}.pager{font-size:12px;font-weight:300;color:#999999}.list-group{font-size:12px;font-weight:300}.close{opacity:0.4}.close:hover,.close:focus{opacity:1}.alert{font-size:12px;font-weight:300}.alert a,.alert .alert-link{font-weight:normal;color:#fff;text-decoration:underline}.alert .close{color:#fff;text-decoration:none}.alert .close:hover,.alert .close:focus{color:#fff}.label{padding-left:1em;padding-right:1em;border-radius:0;font-weight:300}.label-default{background-color:#e7e7e7;color:#333333}.badge{font-weight:300}.progress{height:22px;padding:2px;background-color:#f6f6f6;border:1px solid #ccc;-webkit-box-shadow:none;box-shadow:none}.dropdown-menu{padding:0;margin-top:0;font-size:12px}.dropdown-menu>li>a{padding:12px 15px}.dropdown-header{padding-left:15px;padding-right:15px;font-size:9px;text-transform:uppercase}.popover{color:#fff;font-size:12px;font-weight:300}.panel-heading,.panel-footer{border-top-right-radius:0;border-top-left-radius:0} \ No newline at end of file diff --git a/.rclone_repo/docs/static/css/custom.css b/.rclone_repo/docs/static/css/custom.css new file mode 100755 index 0000000..ec96edd --- /dev/null +++ b/.rclone_repo/docs/static/css/custom.css @@ -0,0 +1,53 @@ +body { + margin-top: 75px; /* 100px is double the height of the navbar - I made it a big larger for some more space - keep it at 50px at least if you want to use the fixed top nav */ +} + +footer { + margin: 50px 0; +} + +table { + background-color:#e0e0ff +} + +tbody td, th { + border: 1px solid black; + padding: 3px 7px 2px 7px; +} + +thead td, th { + border: 1px solid black; + padding: 3px 7px 2px 7px; + font-weight: bold; +} + +tbody tr:nth-child(odd) { + background-color:#d0d0ff +} + +pre code { + /* Preserve whitespace. Wrap text only at line breaks. */ + /* Avoids auto-wrapping the code lines. */ + white-space: pre +} + +/* Hover over links on headers */ +.header-link { + position: absolute; + left: -0.5em; + opacity: 0; + transition: opacity 0.2s ease-in-out 0.1s; +} +h2:hover .header-link, +h3:hover .header-link, +h4:hover .header-link, +h5:hover .header-link, +h6:hover .header-link { + opacity: 1; +} + +/* Fix spacing between menu items */ +.navbar-default .dropdown-menu>li>a { + padding-top: 6px; + padding-bottom: 6px; +} diff --git a/.rclone_repo/docs/static/css/font-awesome.css b/.rclone_repo/docs/static/css/font-awesome.css new file mode 100755 index 0000000..a0b879f --- /dev/null +++ b/.rclone_repo/docs/static/css/font-awesome.css @@ -0,0 +1,2199 @@ +/*! + * Font Awesome 4.6.3 by @davegandy - http://fontawesome.io - @fontawesome + * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) + */ +/* FONT PATH + * -------------------------- */ +@font-face { + font-family: 'FontAwesome'; + src: url('../fonts/fontawesome-webfont.eot?v=4.6.3'); + src: url('../fonts/fontawesome-webfont.eot?#iefix&v=4.6.3') format('embedded-opentype'), url('../fonts/fontawesome-webfont.woff2?v=4.6.3') format('woff2'), url('../fonts/fontawesome-webfont.woff?v=4.6.3') format('woff'), url('../fonts/fontawesome-webfont.ttf?v=4.6.3') format('truetype'), url('../fonts/fontawesome-webfont.svg?v=4.6.3#fontawesomeregular') format('svg'); + font-weight: normal; + font-style: normal; +} +.fa { + display: inline-block; + font: normal normal normal 14px/1 FontAwesome; + font-size: inherit; + text-rendering: auto; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} +/* makes the font 33% larger relative to the icon container */ +.fa-lg { + font-size: 1.33333333em; + line-height: 0.75em; + vertical-align: -15%; +} +.fa-2x { + font-size: 2em; +} +.fa-3x { + font-size: 3em; +} +.fa-4x { + font-size: 4em; +} +.fa-5x { + font-size: 5em; +} +.fa-fw { + width: 1.28571429em; + text-align: center; +} +.fa-ul { + padding-left: 0; + margin-left: 2.14285714em; + list-style-type: none; +} +.fa-ul > li { + position: relative; +} +.fa-li { + position: absolute; + left: -2.14285714em; + width: 2.14285714em; + top: 0.14285714em; + text-align: center; +} +.fa-li.fa-lg { + left: -1.85714286em; +} +.fa-border { + padding: .2em .25em .15em; + border: solid 0.08em #eeeeee; + border-radius: .1em; +} +.fa-pull-left { + float: left; +} +.fa-pull-right { + float: right; +} +.fa.fa-pull-left { + margin-right: .3em; +} +.fa.fa-pull-right { + margin-left: .3em; +} +/* Deprecated as of 4.4.0 */ +.pull-right { + float: right; +} +.pull-left { + float: left; +} +.fa.pull-left { + margin-right: .3em; +} +.fa.pull-right { + margin-left: .3em; +} +.fa-spin { + -webkit-animation: fa-spin 2s infinite linear; + animation: fa-spin 2s infinite linear; +} +.fa-pulse { + -webkit-animation: fa-spin 1s infinite steps(8); + animation: fa-spin 1s infinite steps(8); +} +@-webkit-keyframes fa-spin { + 0% { + -webkit-transform: rotate(0deg); + transform: rotate(0deg); + } + 100% { + -webkit-transform: rotate(359deg); + transform: rotate(359deg); + } +} +@keyframes fa-spin { + 0% { + -webkit-transform: rotate(0deg); + transform: rotate(0deg); + } + 100% { + -webkit-transform: rotate(359deg); + transform: rotate(359deg); + } +} +.fa-rotate-90 { + -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=1)"; + -webkit-transform: rotate(90deg); + -ms-transform: rotate(90deg); + transform: rotate(90deg); +} +.fa-rotate-180 { + -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2)"; + -webkit-transform: rotate(180deg); + -ms-transform: rotate(180deg); + transform: rotate(180deg); +} +.fa-rotate-270 { + -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=3)"; + -webkit-transform: rotate(270deg); + -ms-transform: rotate(270deg); + transform: rotate(270deg); +} +.fa-flip-horizontal { + -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)"; + -webkit-transform: scale(-1, 1); + -ms-transform: scale(-1, 1); + transform: scale(-1, 1); +} +.fa-flip-vertical { + -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)"; + -webkit-transform: scale(1, -1); + -ms-transform: scale(1, -1); + transform: scale(1, -1); +} +:root .fa-rotate-90, +:root .fa-rotate-180, +:root .fa-rotate-270, +:root .fa-flip-horizontal, +:root .fa-flip-vertical { + filter: none; +} +.fa-stack { + position: relative; + display: inline-block; + width: 2em; + height: 2em; + line-height: 2em; + vertical-align: middle; +} +.fa-stack-1x, +.fa-stack-2x { + position: absolute; + left: 0; + width: 100%; + text-align: center; +} +.fa-stack-1x { + line-height: inherit; +} +.fa-stack-2x { + font-size: 2em; +} +.fa-inverse { + color: #ffffff; +} +/* Font Awesome uses the Unicode Private Use Area (PUA) to ensure screen + readers do not read off random characters that represent icons */ +.fa-glass:before { + content: "\f000"; +} +.fa-music:before { + content: "\f001"; +} +.fa-search:before { + content: "\f002"; +} +.fa-envelope-o:before { + content: "\f003"; +} +.fa-heart:before { + content: "\f004"; +} +.fa-star:before { + content: "\f005"; +} +.fa-star-o:before { + content: "\f006"; +} +.fa-user:before { + content: "\f007"; +} +.fa-film:before { + content: "\f008"; +} +.fa-th-large:before { + content: "\f009"; +} +.fa-th:before { + content: "\f00a"; +} +.fa-th-list:before { + content: "\f00b"; +} +.fa-check:before { + content: "\f00c"; +} +.fa-remove:before, +.fa-close:before, +.fa-times:before { + content: "\f00d"; +} +.fa-search-plus:before { + content: "\f00e"; +} +.fa-search-minus:before { + content: "\f010"; +} +.fa-power-off:before { + content: "\f011"; +} +.fa-signal:before { + content: "\f012"; +} +.fa-gear:before, +.fa-cog:before { + content: "\f013"; +} +.fa-trash-o:before { + content: "\f014"; +} +.fa-home:before { + content: "\f015"; +} +.fa-file-o:before { + content: "\f016"; +} +.fa-clock-o:before { + content: "\f017"; +} +.fa-road:before { + content: "\f018"; +} +.fa-download:before { + content: "\f019"; +} +.fa-arrow-circle-o-down:before { + content: "\f01a"; +} +.fa-arrow-circle-o-up:before { + content: "\f01b"; +} +.fa-inbox:before { + content: "\f01c"; +} +.fa-play-circle-o:before { + content: "\f01d"; +} +.fa-rotate-right:before, +.fa-repeat:before { + content: "\f01e"; +} +.fa-refresh:before { + content: "\f021"; +} +.fa-list-alt:before { + content: "\f022"; +} +.fa-lock:before { + content: "\f023"; +} +.fa-flag:before { + content: "\f024"; +} +.fa-headphones:before { + content: "\f025"; +} +.fa-volume-off:before { + content: "\f026"; +} +.fa-volume-down:before { + content: "\f027"; +} +.fa-volume-up:before { + content: "\f028"; +} +.fa-qrcode:before { + content: "\f029"; +} +.fa-barcode:before { + content: "\f02a"; +} +.fa-tag:before { + content: "\f02b"; +} +.fa-tags:before { + content: "\f02c"; +} +.fa-book:before { + content: "\f02d"; +} +.fa-bookmark:before { + content: "\f02e"; +} +.fa-print:before { + content: "\f02f"; +} +.fa-camera:before { + content: "\f030"; +} +.fa-font:before { + content: "\f031"; +} +.fa-bold:before { + content: "\f032"; +} +.fa-italic:before { + content: "\f033"; +} +.fa-text-height:before { + content: "\f034"; +} +.fa-text-width:before { + content: "\f035"; +} +.fa-align-left:before { + content: "\f036"; +} +.fa-align-center:before { + content: "\f037"; +} +.fa-align-right:before { + content: "\f038"; +} +.fa-align-justify:before { + content: "\f039"; +} +.fa-list:before { + content: "\f03a"; +} +.fa-dedent:before, +.fa-outdent:before { + content: "\f03b"; +} +.fa-indent:before { + content: "\f03c"; +} +.fa-video-camera:before { + content: "\f03d"; +} +.fa-photo:before, +.fa-image:before, +.fa-picture-o:before { + content: "\f03e"; +} +.fa-pencil:before { + content: "\f040"; +} +.fa-map-marker:before { + content: "\f041"; +} +.fa-adjust:before { + content: "\f042"; +} +.fa-tint:before { + content: "\f043"; +} +.fa-edit:before, +.fa-pencil-square-o:before { + content: "\f044"; +} +.fa-share-square-o:before { + content: "\f045"; +} +.fa-check-square-o:before { + content: "\f046"; +} +.fa-arrows:before { + content: "\f047"; +} +.fa-step-backward:before { + content: "\f048"; +} +.fa-fast-backward:before { + content: "\f049"; +} +.fa-backward:before { + content: "\f04a"; +} +.fa-play:before { + content: "\f04b"; +} +.fa-pause:before { + content: "\f04c"; +} +.fa-stop:before { + content: "\f04d"; +} +.fa-forward:before { + content: "\f04e"; +} +.fa-fast-forward:before { + content: "\f050"; +} +.fa-step-forward:before { + content: "\f051"; +} +.fa-eject:before { + content: "\f052"; +} +.fa-chevron-left:before { + content: "\f053"; +} +.fa-chevron-right:before { + content: "\f054"; +} +.fa-plus-circle:before { + content: "\f055"; +} +.fa-minus-circle:before { + content: "\f056"; +} +.fa-times-circle:before { + content: "\f057"; +} +.fa-check-circle:before { + content: "\f058"; +} +.fa-question-circle:before { + content: "\f059"; +} +.fa-info-circle:before { + content: "\f05a"; +} +.fa-crosshairs:before { + content: "\f05b"; +} +.fa-times-circle-o:before { + content: "\f05c"; +} +.fa-check-circle-o:before { + content: "\f05d"; +} +.fa-ban:before { + content: "\f05e"; +} +.fa-arrow-left:before { + content: "\f060"; +} +.fa-arrow-right:before { + content: "\f061"; +} +.fa-arrow-up:before { + content: "\f062"; +} +.fa-arrow-down:before { + content: "\f063"; +} +.fa-mail-forward:before, +.fa-share:before { + content: "\f064"; +} +.fa-expand:before { + content: "\f065"; +} +.fa-compress:before { + content: "\f066"; +} +.fa-plus:before { + content: "\f067"; +} +.fa-minus:before { + content: "\f068"; +} +.fa-asterisk:before { + content: "\f069"; +} +.fa-exclamation-circle:before { + content: "\f06a"; +} +.fa-gift:before { + content: "\f06b"; +} +.fa-leaf:before { + content: "\f06c"; +} +.fa-fire:before { + content: "\f06d"; +} +.fa-eye:before { + content: "\f06e"; +} +.fa-eye-slash:before { + content: "\f070"; +} +.fa-warning:before, +.fa-exclamation-triangle:before { + content: "\f071"; +} +.fa-plane:before { + content: "\f072"; +} +.fa-calendar:before { + content: "\f073"; +} +.fa-random:before { + content: "\f074"; +} +.fa-comment:before { + content: "\f075"; +} +.fa-magnet:before { + content: "\f076"; +} +.fa-chevron-up:before { + content: "\f077"; +} +.fa-chevron-down:before { + content: "\f078"; +} +.fa-retweet:before { + content: "\f079"; +} +.fa-shopping-cart:before { + content: "\f07a"; +} +.fa-folder:before { + content: "\f07b"; +} +.fa-folder-open:before { + content: "\f07c"; +} +.fa-arrows-v:before { + content: "\f07d"; +} +.fa-arrows-h:before { + content: "\f07e"; +} +.fa-bar-chart-o:before, +.fa-bar-chart:before { + content: "\f080"; +} +.fa-twitter-square:before { + content: "\f081"; +} +.fa-facebook-square:before { + content: "\f082"; +} +.fa-camera-retro:before { + content: "\f083"; +} +.fa-key:before { + content: "\f084"; +} +.fa-gears:before, +.fa-cogs:before { + content: "\f085"; +} +.fa-comments:before { + content: "\f086"; +} +.fa-thumbs-o-up:before { + content: "\f087"; +} +.fa-thumbs-o-down:before { + content: "\f088"; +} +.fa-star-half:before { + content: "\f089"; +} +.fa-heart-o:before { + content: "\f08a"; +} +.fa-sign-out:before { + content: "\f08b"; +} +.fa-linkedin-square:before { + content: "\f08c"; +} +.fa-thumb-tack:before { + content: "\f08d"; +} +.fa-external-link:before { + content: "\f08e"; +} +.fa-sign-in:before { + content: "\f090"; +} +.fa-trophy:before { + content: "\f091"; +} +.fa-github-square:before { + content: "\f092"; +} +.fa-upload:before { + content: "\f093"; +} +.fa-lemon-o:before { + content: "\f094"; +} +.fa-phone:before { + content: "\f095"; +} +.fa-square-o:before { + content: "\f096"; +} +.fa-bookmark-o:before { + content: "\f097"; +} +.fa-phone-square:before { + content: "\f098"; +} +.fa-twitter:before { + content: "\f099"; +} +.fa-facebook-f:before, +.fa-facebook:before { + content: "\f09a"; +} +.fa-github:before { + content: "\f09b"; +} +.fa-unlock:before { + content: "\f09c"; +} +.fa-credit-card:before { + content: "\f09d"; +} +.fa-feed:before, +.fa-rss:before { + content: "\f09e"; +} +.fa-hdd-o:before { + content: "\f0a0"; +} +.fa-bullhorn:before { + content: "\f0a1"; +} +.fa-bell:before { + content: "\f0f3"; +} +.fa-certificate:before { + content: "\f0a3"; +} +.fa-hand-o-right:before { + content: "\f0a4"; +} +.fa-hand-o-left:before { + content: "\f0a5"; +} +.fa-hand-o-up:before { + content: "\f0a6"; +} +.fa-hand-o-down:before { + content: "\f0a7"; +} +.fa-arrow-circle-left:before { + content: "\f0a8"; +} +.fa-arrow-circle-right:before { + content: "\f0a9"; +} +.fa-arrow-circle-up:before { + content: "\f0aa"; +} +.fa-arrow-circle-down:before { + content: "\f0ab"; +} +.fa-globe:before { + content: "\f0ac"; +} +.fa-wrench:before { + content: "\f0ad"; +} +.fa-tasks:before { + content: "\f0ae"; +} +.fa-filter:before { + content: "\f0b0"; +} +.fa-briefcase:before { + content: "\f0b1"; +} +.fa-arrows-alt:before { + content: "\f0b2"; +} +.fa-group:before, +.fa-users:before { + content: "\f0c0"; +} +.fa-chain:before, +.fa-link:before { + content: "\f0c1"; +} +.fa-cloud:before { + content: "\f0c2"; +} +.fa-flask:before { + content: "\f0c3"; +} +.fa-cut:before, +.fa-scissors:before { + content: "\f0c4"; +} +.fa-copy:before, +.fa-files-o:before { + content: "\f0c5"; +} +.fa-paperclip:before { + content: "\f0c6"; +} +.fa-save:before, +.fa-floppy-o:before { + content: "\f0c7"; +} +.fa-square:before { + content: "\f0c8"; +} +.fa-navicon:before, +.fa-reorder:before, +.fa-bars:before { + content: "\f0c9"; +} +.fa-list-ul:before { + content: "\f0ca"; +} +.fa-list-ol:before { + content: "\f0cb"; +} +.fa-strikethrough:before { + content: "\f0cc"; +} +.fa-underline:before { + content: "\f0cd"; +} +.fa-table:before { + content: "\f0ce"; +} +.fa-magic:before { + content: "\f0d0"; +} +.fa-truck:before { + content: "\f0d1"; +} +.fa-pinterest:before { + content: "\f0d2"; +} +.fa-pinterest-square:before { + content: "\f0d3"; +} +.fa-google-plus-square:before { + content: "\f0d4"; +} +.fa-google-plus:before { + content: "\f0d5"; +} +.fa-money:before { + content: "\f0d6"; +} +.fa-caret-down:before { + content: "\f0d7"; +} +.fa-caret-up:before { + content: "\f0d8"; +} +.fa-caret-left:before { + content: "\f0d9"; +} +.fa-caret-right:before { + content: "\f0da"; +} +.fa-columns:before { + content: "\f0db"; +} +.fa-unsorted:before, +.fa-sort:before { + content: "\f0dc"; +} +.fa-sort-down:before, +.fa-sort-desc:before { + content: "\f0dd"; +} +.fa-sort-up:before, +.fa-sort-asc:before { + content: "\f0de"; +} +.fa-envelope:before { + content: "\f0e0"; +} +.fa-linkedin:before { + content: "\f0e1"; +} +.fa-rotate-left:before, +.fa-undo:before { + content: "\f0e2"; +} +.fa-legal:before, +.fa-gavel:before { + content: "\f0e3"; +} +.fa-dashboard:before, +.fa-tachometer:before { + content: "\f0e4"; +} +.fa-comment-o:before { + content: "\f0e5"; +} +.fa-comments-o:before { + content: "\f0e6"; +} +.fa-flash:before, +.fa-bolt:before { + content: "\f0e7"; +} +.fa-sitemap:before { + content: "\f0e8"; +} +.fa-umbrella:before { + content: "\f0e9"; +} +.fa-paste:before, +.fa-clipboard:before { + content: "\f0ea"; +} +.fa-lightbulb-o:before { + content: "\f0eb"; +} +.fa-exchange:before { + content: "\f0ec"; +} +.fa-cloud-download:before { + content: "\f0ed"; +} +.fa-cloud-upload:before { + content: "\f0ee"; +} +.fa-user-md:before { + content: "\f0f0"; +} +.fa-stethoscope:before { + content: "\f0f1"; +} +.fa-suitcase:before { + content: "\f0f2"; +} +.fa-bell-o:before { + content: "\f0a2"; +} +.fa-coffee:before { + content: "\f0f4"; +} +.fa-cutlery:before { + content: "\f0f5"; +} +.fa-file-text-o:before { + content: "\f0f6"; +} +.fa-building-o:before { + content: "\f0f7"; +} +.fa-hospital-o:before { + content: "\f0f8"; +} +.fa-ambulance:before { + content: "\f0f9"; +} +.fa-medkit:before { + content: "\f0fa"; +} +.fa-fighter-jet:before { + content: "\f0fb"; +} +.fa-beer:before { + content: "\f0fc"; +} +.fa-h-square:before { + content: "\f0fd"; +} +.fa-plus-square:before { + content: "\f0fe"; +} +.fa-angle-double-left:before { + content: "\f100"; +} +.fa-angle-double-right:before { + content: "\f101"; +} +.fa-angle-double-up:before { + content: "\f102"; +} +.fa-angle-double-down:before { + content: "\f103"; +} +.fa-angle-left:before { + content: "\f104"; +} +.fa-angle-right:before { + content: "\f105"; +} +.fa-angle-up:before { + content: "\f106"; +} +.fa-angle-down:before { + content: "\f107"; +} +.fa-desktop:before { + content: "\f108"; +} +.fa-laptop:before { + content: "\f109"; +} +.fa-tablet:before { + content: "\f10a"; +} +.fa-mobile-phone:before, +.fa-mobile:before { + content: "\f10b"; +} +.fa-circle-o:before { + content: "\f10c"; +} +.fa-quote-left:before { + content: "\f10d"; +} +.fa-quote-right:before { + content: "\f10e"; +} +.fa-spinner:before { + content: "\f110"; +} +.fa-circle:before { + content: "\f111"; +} +.fa-mail-reply:before, +.fa-reply:before { + content: "\f112"; +} +.fa-github-alt:before { + content: "\f113"; +} +.fa-folder-o:before { + content: "\f114"; +} +.fa-folder-open-o:before { + content: "\f115"; +} +.fa-smile-o:before { + content: "\f118"; +} +.fa-frown-o:before { + content: "\f119"; +} +.fa-meh-o:before { + content: "\f11a"; +} +.fa-gamepad:before { + content: "\f11b"; +} +.fa-keyboard-o:before { + content: "\f11c"; +} +.fa-flag-o:before { + content: "\f11d"; +} +.fa-flag-checkered:before { + content: "\f11e"; +} +.fa-terminal:before { + content: "\f120"; +} +.fa-code:before { + content: "\f121"; +} +.fa-mail-reply-all:before, +.fa-reply-all:before { + content: "\f122"; +} +.fa-star-half-empty:before, +.fa-star-half-full:before, +.fa-star-half-o:before { + content: "\f123"; +} +.fa-location-arrow:before { + content: "\f124"; +} +.fa-crop:before { + content: "\f125"; +} +.fa-code-fork:before { + content: "\f126"; +} +.fa-unlink:before, +.fa-chain-broken:before { + content: "\f127"; +} +.fa-question:before { + content: "\f128"; +} +.fa-info:before { + content: "\f129"; +} +.fa-exclamation:before { + content: "\f12a"; +} +.fa-superscript:before { + content: "\f12b"; +} +.fa-subscript:before { + content: "\f12c"; +} +.fa-eraser:before { + content: "\f12d"; +} +.fa-puzzle-piece:before { + content: "\f12e"; +} +.fa-microphone:before { + content: "\f130"; +} +.fa-microphone-slash:before { + content: "\f131"; +} +.fa-shield:before { + content: "\f132"; +} +.fa-calendar-o:before { + content: "\f133"; +} +.fa-fire-extinguisher:before { + content: "\f134"; +} +.fa-rocket:before { + content: "\f135"; +} +.fa-maxcdn:before { + content: "\f136"; +} +.fa-chevron-circle-left:before { + content: "\f137"; +} +.fa-chevron-circle-right:before { + content: "\f138"; +} +.fa-chevron-circle-up:before { + content: "\f139"; +} +.fa-chevron-circle-down:before { + content: "\f13a"; +} +.fa-html5:before { + content: "\f13b"; +} +.fa-css3:before { + content: "\f13c"; +} +.fa-anchor:before { + content: "\f13d"; +} +.fa-unlock-alt:before { + content: "\f13e"; +} +.fa-bullseye:before { + content: "\f140"; +} +.fa-ellipsis-h:before { + content: "\f141"; +} +.fa-ellipsis-v:before { + content: "\f142"; +} +.fa-rss-square:before { + content: "\f143"; +} +.fa-play-circle:before { + content: "\f144"; +} +.fa-ticket:before { + content: "\f145"; +} +.fa-minus-square:before { + content: "\f146"; +} +.fa-minus-square-o:before { + content: "\f147"; +} +.fa-level-up:before { + content: "\f148"; +} +.fa-level-down:before { + content: "\f149"; +} +.fa-check-square:before { + content: "\f14a"; +} +.fa-pencil-square:before { + content: "\f14b"; +} +.fa-external-link-square:before { + content: "\f14c"; +} +.fa-share-square:before { + content: "\f14d"; +} +.fa-compass:before { + content: "\f14e"; +} +.fa-toggle-down:before, +.fa-caret-square-o-down:before { + content: "\f150"; +} +.fa-toggle-up:before, +.fa-caret-square-o-up:before { + content: "\f151"; +} +.fa-toggle-right:before, +.fa-caret-square-o-right:before { + content: "\f152"; +} +.fa-euro:before, +.fa-eur:before { + content: "\f153"; +} +.fa-gbp:before { + content: "\f154"; +} +.fa-dollar:before, +.fa-usd:before { + content: "\f155"; +} +.fa-rupee:before, +.fa-inr:before { + content: "\f156"; +} +.fa-cny:before, +.fa-rmb:before, +.fa-yen:before, +.fa-jpy:before { + content: "\f157"; +} +.fa-ruble:before, +.fa-rouble:before, +.fa-rub:before { + content: "\f158"; +} +.fa-won:before, +.fa-krw:before { + content: "\f159"; +} +.fa-bitcoin:before, +.fa-btc:before { + content: "\f15a"; +} +.fa-file:before { + content: "\f15b"; +} +.fa-file-text:before { + content: "\f15c"; +} +.fa-sort-alpha-asc:before { + content: "\f15d"; +} +.fa-sort-alpha-desc:before { + content: "\f15e"; +} +.fa-sort-amount-asc:before { + content: "\f160"; +} +.fa-sort-amount-desc:before { + content: "\f161"; +} +.fa-sort-numeric-asc:before { + content: "\f162"; +} +.fa-sort-numeric-desc:before { + content: "\f163"; +} +.fa-thumbs-up:before { + content: "\f164"; +} +.fa-thumbs-down:before { + content: "\f165"; +} +.fa-youtube-square:before { + content: "\f166"; +} +.fa-youtube:before { + content: "\f167"; +} +.fa-xing:before { + content: "\f168"; +} +.fa-xing-square:before { + content: "\f169"; +} +.fa-youtube-play:before { + content: "\f16a"; +} +.fa-dropbox:before { + content: "\f16b"; +} +.fa-stack-overflow:before { + content: "\f16c"; +} +.fa-instagram:before { + content: "\f16d"; +} +.fa-flickr:before { + content: "\f16e"; +} +.fa-adn:before { + content: "\f170"; +} +.fa-bitbucket:before { + content: "\f171"; +} +.fa-bitbucket-square:before { + content: "\f172"; +} +.fa-tumblr:before { + content: "\f173"; +} +.fa-tumblr-square:before { + content: "\f174"; +} +.fa-long-arrow-down:before { + content: "\f175"; +} +.fa-long-arrow-up:before { + content: "\f176"; +} +.fa-long-arrow-left:before { + content: "\f177"; +} +.fa-long-arrow-right:before { + content: "\f178"; +} +.fa-apple:before { + content: "\f179"; +} +.fa-windows:before { + content: "\f17a"; +} +.fa-android:before { + content: "\f17b"; +} +.fa-linux:before { + content: "\f17c"; +} +.fa-dribbble:before { + content: "\f17d"; +} +.fa-skype:before { + content: "\f17e"; +} +.fa-foursquare:before { + content: "\f180"; +} +.fa-trello:before { + content: "\f181"; +} +.fa-female:before { + content: "\f182"; +} +.fa-male:before { + content: "\f183"; +} +.fa-gittip:before, +.fa-gratipay:before { + content: "\f184"; +} +.fa-sun-o:before { + content: "\f185"; +} +.fa-moon-o:before { + content: "\f186"; +} +.fa-archive:before { + content: "\f187"; +} +.fa-bug:before { + content: "\f188"; +} +.fa-vk:before { + content: "\f189"; +} +.fa-weibo:before { + content: "\f18a"; +} +.fa-renren:before { + content: "\f18b"; +} +.fa-pagelines:before { + content: "\f18c"; +} +.fa-stack-exchange:before { + content: "\f18d"; +} +.fa-arrow-circle-o-right:before { + content: "\f18e"; +} +.fa-arrow-circle-o-left:before { + content: "\f190"; +} +.fa-toggle-left:before, +.fa-caret-square-o-left:before { + content: "\f191"; +} +.fa-dot-circle-o:before { + content: "\f192"; +} +.fa-wheelchair:before { + content: "\f193"; +} +.fa-vimeo-square:before { + content: "\f194"; +} +.fa-turkish-lira:before, +.fa-try:before { + content: "\f195"; +} +.fa-plus-square-o:before { + content: "\f196"; +} +.fa-space-shuttle:before { + content: "\f197"; +} +.fa-slack:before { + content: "\f198"; +} +.fa-envelope-square:before { + content: "\f199"; +} +.fa-wordpress:before { + content: "\f19a"; +} +.fa-openid:before { + content: "\f19b"; +} +.fa-institution:before, +.fa-bank:before, +.fa-university:before { + content: "\f19c"; +} +.fa-mortar-board:before, +.fa-graduation-cap:before { + content: "\f19d"; +} +.fa-yahoo:before { + content: "\f19e"; +} +.fa-google:before { + content: "\f1a0"; +} +.fa-reddit:before { + content: "\f1a1"; +} +.fa-reddit-square:before { + content: "\f1a2"; +} +.fa-stumbleupon-circle:before { + content: "\f1a3"; +} +.fa-stumbleupon:before { + content: "\f1a4"; +} +.fa-delicious:before { + content: "\f1a5"; +} +.fa-digg:before { + content: "\f1a6"; +} +.fa-pied-piper-pp:before { + content: "\f1a7"; +} +.fa-pied-piper-alt:before { + content: "\f1a8"; +} +.fa-drupal:before { + content: "\f1a9"; +} +.fa-joomla:before { + content: "\f1aa"; +} +.fa-language:before { + content: "\f1ab"; +} +.fa-fax:before { + content: "\f1ac"; +} +.fa-building:before { + content: "\f1ad"; +} +.fa-child:before { + content: "\f1ae"; +} +.fa-paw:before { + content: "\f1b0"; +} +.fa-spoon:before { + content: "\f1b1"; +} +.fa-cube:before { + content: "\f1b2"; +} +.fa-cubes:before { + content: "\f1b3"; +} +.fa-behance:before { + content: "\f1b4"; +} +.fa-behance-square:before { + content: "\f1b5"; +} +.fa-steam:before { + content: "\f1b6"; +} +.fa-steam-square:before { + content: "\f1b7"; +} +.fa-recycle:before { + content: "\f1b8"; +} +.fa-automobile:before, +.fa-car:before { + content: "\f1b9"; +} +.fa-cab:before, +.fa-taxi:before { + content: "\f1ba"; +} +.fa-tree:before { + content: "\f1bb"; +} +.fa-spotify:before { + content: "\f1bc"; +} +.fa-deviantart:before { + content: "\f1bd"; +} +.fa-soundcloud:before { + content: "\f1be"; +} +.fa-database:before { + content: "\f1c0"; +} +.fa-file-pdf-o:before { + content: "\f1c1"; +} +.fa-file-word-o:before { + content: "\f1c2"; +} +.fa-file-excel-o:before { + content: "\f1c3"; +} +.fa-file-powerpoint-o:before { + content: "\f1c4"; +} +.fa-file-photo-o:before, +.fa-file-picture-o:before, +.fa-file-image-o:before { + content: "\f1c5"; +} +.fa-file-zip-o:before, +.fa-file-archive-o:before { + content: "\f1c6"; +} +.fa-file-sound-o:before, +.fa-file-audio-o:before { + content: "\f1c7"; +} +.fa-file-movie-o:before, +.fa-file-video-o:before { + content: "\f1c8"; +} +.fa-file-code-o:before { + content: "\f1c9"; +} +.fa-vine:before { + content: "\f1ca"; +} +.fa-codepen:before { + content: "\f1cb"; +} +.fa-jsfiddle:before { + content: "\f1cc"; +} +.fa-life-bouy:before, +.fa-life-buoy:before, +.fa-life-saver:before, +.fa-support:before, +.fa-life-ring:before { + content: "\f1cd"; +} +.fa-circle-o-notch:before { + content: "\f1ce"; +} +.fa-ra:before, +.fa-resistance:before, +.fa-rebel:before { + content: "\f1d0"; +} +.fa-ge:before, +.fa-empire:before { + content: "\f1d1"; +} +.fa-git-square:before { + content: "\f1d2"; +} +.fa-git:before { + content: "\f1d3"; +} +.fa-y-combinator-square:before, +.fa-yc-square:before, +.fa-hacker-news:before { + content: "\f1d4"; +} +.fa-tencent-weibo:before { + content: "\f1d5"; +} +.fa-qq:before { + content: "\f1d6"; +} +.fa-wechat:before, +.fa-weixin:before { + content: "\f1d7"; +} +.fa-send:before, +.fa-paper-plane:before { + content: "\f1d8"; +} +.fa-send-o:before, +.fa-paper-plane-o:before { + content: "\f1d9"; +} +.fa-history:before { + content: "\f1da"; +} +.fa-circle-thin:before { + content: "\f1db"; +} +.fa-header:before { + content: "\f1dc"; +} +.fa-paragraph:before { + content: "\f1dd"; +} +.fa-sliders:before { + content: "\f1de"; +} +.fa-share-alt:before { + content: "\f1e0"; +} +.fa-share-alt-square:before { + content: "\f1e1"; +} +.fa-bomb:before { + content: "\f1e2"; +} +.fa-soccer-ball-o:before, +.fa-futbol-o:before { + content: "\f1e3"; +} +.fa-tty:before { + content: "\f1e4"; +} +.fa-binoculars:before { + content: "\f1e5"; +} +.fa-plug:before { + content: "\f1e6"; +} +.fa-slideshare:before { + content: "\f1e7"; +} +.fa-twitch:before { + content: "\f1e8"; +} +.fa-yelp:before { + content: "\f1e9"; +} +.fa-newspaper-o:before { + content: "\f1ea"; +} +.fa-wifi:before { + content: "\f1eb"; +} +.fa-calculator:before { + content: "\f1ec"; +} +.fa-paypal:before { + content: "\f1ed"; +} +.fa-google-wallet:before { + content: "\f1ee"; +} +.fa-cc-visa:before { + content: "\f1f0"; +} +.fa-cc-mastercard:before { + content: "\f1f1"; +} +.fa-cc-discover:before { + content: "\f1f2"; +} +.fa-cc-amex:before { + content: "\f1f3"; +} +.fa-cc-paypal:before { + content: "\f1f4"; +} +.fa-cc-stripe:before { + content: "\f1f5"; +} +.fa-bell-slash:before { + content: "\f1f6"; +} +.fa-bell-slash-o:before { + content: "\f1f7"; +} +.fa-trash:before { + content: "\f1f8"; +} +.fa-copyright:before { + content: "\f1f9"; +} +.fa-at:before { + content: "\f1fa"; +} +.fa-eyedropper:before { + content: "\f1fb"; +} +.fa-paint-brush:before { + content: "\f1fc"; +} +.fa-birthday-cake:before { + content: "\f1fd"; +} +.fa-area-chart:before { + content: "\f1fe"; +} +.fa-pie-chart:before { + content: "\f200"; +} +.fa-line-chart:before { + content: "\f201"; +} +.fa-lastfm:before { + content: "\f202"; +} +.fa-lastfm-square:before { + content: "\f203"; +} +.fa-toggle-off:before { + content: "\f204"; +} +.fa-toggle-on:before { + content: "\f205"; +} +.fa-bicycle:before { + content: "\f206"; +} +.fa-bus:before { + content: "\f207"; +} +.fa-ioxhost:before { + content: "\f208"; +} +.fa-angellist:before { + content: "\f209"; +} +.fa-cc:before { + content: "\f20a"; +} +.fa-shekel:before, +.fa-sheqel:before, +.fa-ils:before { + content: "\f20b"; +} +.fa-meanpath:before { + content: "\f20c"; +} +.fa-buysellads:before { + content: "\f20d"; +} +.fa-connectdevelop:before { + content: "\f20e"; +} +.fa-dashcube:before { + content: "\f210"; +} +.fa-forumbee:before { + content: "\f211"; +} +.fa-leanpub:before { + content: "\f212"; +} +.fa-sellsy:before { + content: "\f213"; +} +.fa-shirtsinbulk:before { + content: "\f214"; +} +.fa-simplybuilt:before { + content: "\f215"; +} +.fa-skyatlas:before { + content: "\f216"; +} +.fa-cart-plus:before { + content: "\f217"; +} +.fa-cart-arrow-down:before { + content: "\f218"; +} +.fa-diamond:before { + content: "\f219"; +} +.fa-ship:before { + content: "\f21a"; +} +.fa-user-secret:before { + content: "\f21b"; +} +.fa-motorcycle:before { + content: "\f21c"; +} +.fa-street-view:before { + content: "\f21d"; +} +.fa-heartbeat:before { + content: "\f21e"; +} +.fa-venus:before { + content: "\f221"; +} +.fa-mars:before { + content: "\f222"; +} +.fa-mercury:before { + content: "\f223"; +} +.fa-intersex:before, +.fa-transgender:before { + content: "\f224"; +} +.fa-transgender-alt:before { + content: "\f225"; +} +.fa-venus-double:before { + content: "\f226"; +} +.fa-mars-double:before { + content: "\f227"; +} +.fa-venus-mars:before { + content: "\f228"; +} +.fa-mars-stroke:before { + content: "\f229"; +} +.fa-mars-stroke-v:before { + content: "\f22a"; +} +.fa-mars-stroke-h:before { + content: "\f22b"; +} +.fa-neuter:before { + content: "\f22c"; +} +.fa-genderless:before { + content: "\f22d"; +} +.fa-facebook-official:before { + content: "\f230"; +} +.fa-pinterest-p:before { + content: "\f231"; +} +.fa-whatsapp:before { + content: "\f232"; +} +.fa-server:before { + content: "\f233"; +} +.fa-user-plus:before { + content: "\f234"; +} +.fa-user-times:before { + content: "\f235"; +} +.fa-hotel:before, +.fa-bed:before { + content: "\f236"; +} +.fa-viacoin:before { + content: "\f237"; +} +.fa-train:before { + content: "\f238"; +} +.fa-subway:before { + content: "\f239"; +} +.fa-medium:before { + content: "\f23a"; +} +.fa-yc:before, +.fa-y-combinator:before { + content: "\f23b"; +} +.fa-optin-monster:before { + content: "\f23c"; +} +.fa-opencart:before { + content: "\f23d"; +} +.fa-expeditedssl:before { + content: "\f23e"; +} +.fa-battery-4:before, +.fa-battery-full:before { + content: "\f240"; +} +.fa-battery-3:before, +.fa-battery-three-quarters:before { + content: "\f241"; +} +.fa-battery-2:before, +.fa-battery-half:before { + content: "\f242"; +} +.fa-battery-1:before, +.fa-battery-quarter:before { + content: "\f243"; +} +.fa-battery-0:before, +.fa-battery-empty:before { + content: "\f244"; +} +.fa-mouse-pointer:before { + content: "\f245"; +} +.fa-i-cursor:before { + content: "\f246"; +} +.fa-object-group:before { + content: "\f247"; +} +.fa-object-ungroup:before { + content: "\f248"; +} +.fa-sticky-note:before { + content: "\f249"; +} +.fa-sticky-note-o:before { + content: "\f24a"; +} +.fa-cc-jcb:before { + content: "\f24b"; +} +.fa-cc-diners-club:before { + content: "\f24c"; +} +.fa-clone:before { + content: "\f24d"; +} +.fa-balance-scale:before { + content: "\f24e"; +} +.fa-hourglass-o:before { + content: "\f250"; +} +.fa-hourglass-1:before, +.fa-hourglass-start:before { + content: "\f251"; +} +.fa-hourglass-2:before, +.fa-hourglass-half:before { + content: "\f252"; +} +.fa-hourglass-3:before, +.fa-hourglass-end:before { + content: "\f253"; +} +.fa-hourglass:before { + content: "\f254"; +} +.fa-hand-grab-o:before, +.fa-hand-rock-o:before { + content: "\f255"; +} +.fa-hand-stop-o:before, +.fa-hand-paper-o:before { + content: "\f256"; +} +.fa-hand-scissors-o:before { + content: "\f257"; +} +.fa-hand-lizard-o:before { + content: "\f258"; +} +.fa-hand-spock-o:before { + content: "\f259"; +} +.fa-hand-pointer-o:before { + content: "\f25a"; +} +.fa-hand-peace-o:before { + content: "\f25b"; +} +.fa-trademark:before { + content: "\f25c"; +} +.fa-registered:before { + content: "\f25d"; +} +.fa-creative-commons:before { + content: "\f25e"; +} +.fa-gg:before { + content: "\f260"; +} +.fa-gg-circle:before { + content: "\f261"; +} +.fa-tripadvisor:before { + content: "\f262"; +} +.fa-odnoklassniki:before { + content: "\f263"; +} +.fa-odnoklassniki-square:before { + content: "\f264"; +} +.fa-get-pocket:before { + content: "\f265"; +} +.fa-wikipedia-w:before { + content: "\f266"; +} +.fa-safari:before { + content: "\f267"; +} +.fa-chrome:before { + content: "\f268"; +} +.fa-firefox:before { + content: "\f269"; +} +.fa-opera:before { + content: "\f26a"; +} +.fa-internet-explorer:before { + content: "\f26b"; +} +.fa-tv:before, +.fa-television:before { + content: "\f26c"; +} +.fa-contao:before { + content: "\f26d"; +} +.fa-500px:before { + content: "\f26e"; +} +.fa-amazon:before { + content: "\f270"; +} +.fa-calendar-plus-o:before { + content: "\f271"; +} +.fa-calendar-minus-o:before { + content: "\f272"; +} +.fa-calendar-times-o:before { + content: "\f273"; +} +.fa-calendar-check-o:before { + content: "\f274"; +} +.fa-industry:before { + content: "\f275"; +} +.fa-map-pin:before { + content: "\f276"; +} +.fa-map-signs:before { + content: "\f277"; +} +.fa-map-o:before { + content: "\f278"; +} +.fa-map:before { + content: "\f279"; +} +.fa-commenting:before { + content: "\f27a"; +} +.fa-commenting-o:before { + content: "\f27b"; +} +.fa-houzz:before { + content: "\f27c"; +} +.fa-vimeo:before { + content: "\f27d"; +} +.fa-black-tie:before { + content: "\f27e"; +} +.fa-fonticons:before { + content: "\f280"; +} +.fa-reddit-alien:before { + content: "\f281"; +} +.fa-edge:before { + content: "\f282"; +} +.fa-credit-card-alt:before { + content: "\f283"; +} +.fa-codiepie:before { + content: "\f284"; +} +.fa-modx:before { + content: "\f285"; +} +.fa-fort-awesome:before { + content: "\f286"; +} +.fa-usb:before { + content: "\f287"; +} +.fa-product-hunt:before { + content: "\f288"; +} +.fa-mixcloud:before { + content: "\f289"; +} +.fa-scribd:before { + content: "\f28a"; +} +.fa-pause-circle:before { + content: "\f28b"; +} +.fa-pause-circle-o:before { + content: "\f28c"; +} +.fa-stop-circle:before { + content: "\f28d"; +} +.fa-stop-circle-o:before { + content: "\f28e"; +} +.fa-shopping-bag:before { + content: "\f290"; +} +.fa-shopping-basket:before { + content: "\f291"; +} +.fa-hashtag:before { + content: "\f292"; +} +.fa-bluetooth:before { + content: "\f293"; +} +.fa-bluetooth-b:before { + content: "\f294"; +} +.fa-percent:before { + content: "\f295"; +} +.fa-gitlab:before { + content: "\f296"; +} +.fa-wpbeginner:before { + content: "\f297"; +} +.fa-wpforms:before { + content: "\f298"; +} +.fa-envira:before { + content: "\f299"; +} +.fa-universal-access:before { + content: "\f29a"; +} +.fa-wheelchair-alt:before { + content: "\f29b"; +} +.fa-question-circle-o:before { + content: "\f29c"; +} +.fa-blind:before { + content: "\f29d"; +} +.fa-audio-description:before { + content: "\f29e"; +} +.fa-volume-control-phone:before { + content: "\f2a0"; +} +.fa-braille:before { + content: "\f2a1"; +} +.fa-assistive-listening-systems:before { + content: "\f2a2"; +} +.fa-asl-interpreting:before, +.fa-american-sign-language-interpreting:before { + content: "\f2a3"; +} +.fa-deafness:before, +.fa-hard-of-hearing:before, +.fa-deaf:before { + content: "\f2a4"; +} +.fa-glide:before { + content: "\f2a5"; +} +.fa-glide-g:before { + content: "\f2a6"; +} +.fa-signing:before, +.fa-sign-language:before { + content: "\f2a7"; +} +.fa-low-vision:before { + content: "\f2a8"; +} +.fa-viadeo:before { + content: "\f2a9"; +} +.fa-viadeo-square:before { + content: "\f2aa"; +} +.fa-snapchat:before { + content: "\f2ab"; +} +.fa-snapchat-ghost:before { + content: "\f2ac"; +} +.fa-snapchat-square:before { + content: "\f2ad"; +} +.fa-pied-piper:before { + content: "\f2ae"; +} +.fa-first-order:before { + content: "\f2b0"; +} +.fa-yoast:before { + content: "\f2b1"; +} +.fa-themeisle:before { + content: "\f2b2"; +} +.fa-google-plus-circle:before, +.fa-google-plus-official:before { + content: "\f2b3"; +} +.fa-fa:before, +.fa-font-awesome:before { + content: "\f2b4"; +} +.sr-only { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + border: 0; +} +.sr-only-focusable:active, +.sr-only-focusable:focus { + position: static; + width: auto; + height: auto; + margin: 0; + overflow: visible; + clip: auto; +} diff --git a/.rclone_repo/docs/static/fonts/FontAwesome.otf b/.rclone_repo/docs/static/fonts/FontAwesome.otf new file mode 100755 index 0000000..d4de13e Binary files /dev/null and b/.rclone_repo/docs/static/fonts/FontAwesome.otf differ diff --git a/.rclone_repo/docs/static/fonts/fontawesome-webfont.eot b/.rclone_repo/docs/static/fonts/fontawesome-webfont.eot new file mode 100755 index 0000000..c7b00d2 Binary files /dev/null and b/.rclone_repo/docs/static/fonts/fontawesome-webfont.eot differ diff --git a/.rclone_repo/docs/static/fonts/fontawesome-webfont.svg b/.rclone_repo/docs/static/fonts/fontawesome-webfont.svg new file mode 100755 index 0000000..8b66187 --- /dev/null +++ b/.rclone_repo/docs/static/fonts/fontawesome-webfont.svg @@ -0,0 +1,685 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.rclone_repo/docs/static/fonts/fontawesome-webfont.ttf b/.rclone_repo/docs/static/fonts/fontawesome-webfont.ttf new file mode 100755 index 0000000..f221e50 Binary files /dev/null and b/.rclone_repo/docs/static/fonts/fontawesome-webfont.ttf differ diff --git a/.rclone_repo/docs/static/fonts/fontawesome-webfont.woff b/.rclone_repo/docs/static/fonts/fontawesome-webfont.woff new file mode 100755 index 0000000..6e7483c Binary files /dev/null and b/.rclone_repo/docs/static/fonts/fontawesome-webfont.woff differ diff --git a/.rclone_repo/docs/static/fonts/fontawesome-webfont.woff2 b/.rclone_repo/docs/static/fonts/fontawesome-webfont.woff2 new file mode 100755 index 0000000..7eb74fd Binary files /dev/null and b/.rclone_repo/docs/static/fonts/fontawesome-webfont.woff2 differ diff --git a/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.eot b/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.eot new file mode 100755 index 0000000..4a4ca86 Binary files /dev/null and b/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.eot differ diff --git a/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.svg b/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.svg new file mode 100755 index 0000000..e3e2dc7 --- /dev/null +++ b/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.svg @@ -0,0 +1,229 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.ttf b/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.ttf new file mode 100755 index 0000000..67fa00b Binary files /dev/null and b/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.ttf differ diff --git a/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.woff b/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.woff new file mode 100755 index 0000000..8c54182 Binary files /dev/null and b/.rclone_repo/docs/static/fonts/glyphicons-halflings-regular.woff differ diff --git a/.rclone_repo/docs/static/img/ncw-bitcoin-address.png b/.rclone_repo/docs/static/img/ncw-bitcoin-address.png new file mode 100755 index 0000000..5c3e3c3 Binary files /dev/null and b/.rclone_repo/docs/static/img/ncw-bitcoin-address.png differ diff --git a/.rclone_repo/docs/static/img/rclone-120x120.png b/.rclone_repo/docs/static/img/rclone-120x120.png new file mode 100755 index 0000000..2d34377 Binary files /dev/null and b/.rclone_repo/docs/static/img/rclone-120x120.png differ diff --git a/.rclone_repo/docs/static/img/rclone-16x16.png b/.rclone_repo/docs/static/img/rclone-16x16.png new file mode 100755 index 0000000..cc129ad Binary files /dev/null and b/.rclone_repo/docs/static/img/rclone-16x16.png differ diff --git a/.rclone_repo/docs/static/img/rclone-32x32.png b/.rclone_repo/docs/static/img/rclone-32x32.png new file mode 100755 index 0000000..38eb3a4 Binary files /dev/null and b/.rclone_repo/docs/static/img/rclone-32x32.png differ diff --git a/.rclone_repo/docs/static/js/bootstrap.js b/.rclone_repo/docs/static/js/bootstrap.js new file mode 100755 index 0000000..8ae571b --- /dev/null +++ b/.rclone_repo/docs/static/js/bootstrap.js @@ -0,0 +1,1951 @@ +/*! + * Bootstrap v3.1.1 (http://getbootstrap.com) + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + */ + +if (typeof jQuery === 'undefined') { throw new Error('Bootstrap\'s JavaScript requires jQuery') } + +/* ======================================================================== + * Bootstrap: transition.js v3.1.1 + * http://getbootstrap.com/javascript/#transitions + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // CSS TRANSITION SUPPORT (Shoutout: http://www.modernizr.com/) + // ============================================================ + + function transitionEnd() { + var el = document.createElement('bootstrap') + + var transEndEventNames = { + 'WebkitTransition' : 'webkitTransitionEnd', + 'MozTransition' : 'transitionend', + 'OTransition' : 'oTransitionEnd otransitionend', + 'transition' : 'transitionend' + } + + for (var name in transEndEventNames) { + if (el.style[name] !== undefined) { + return { end: transEndEventNames[name] } + } + } + + return false // explicit for ie8 ( ._.) + } + + // http://blog.alexmaccaw.com/css-transitions + $.fn.emulateTransitionEnd = function (duration) { + var called = false, $el = this + $(this).one($.support.transition.end, function () { called = true }) + var callback = function () { if (!called) $($el).trigger($.support.transition.end) } + setTimeout(callback, duration) + return this + } + + $(function () { + $.support.transition = transitionEnd() + }) + +}(jQuery); + +/* ======================================================================== + * Bootstrap: alert.js v3.1.1 + * http://getbootstrap.com/javascript/#alerts + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // ALERT CLASS DEFINITION + // ====================== + + var dismiss = '[data-dismiss="alert"]' + var Alert = function (el) { + $(el).on('click', dismiss, this.close) + } + + Alert.prototype.close = function (e) { + var $this = $(this) + var selector = $this.attr('data-target') + + if (!selector) { + selector = $this.attr('href') + selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') // strip for ie7 + } + + var $parent = $(selector) + + if (e) e.preventDefault() + + if (!$parent.length) { + $parent = $this.hasClass('alert') ? $this : $this.parent() + } + + $parent.trigger(e = $.Event('close.bs.alert')) + + if (e.isDefaultPrevented()) return + + $parent.removeClass('in') + + function removeElement() { + $parent.trigger('closed.bs.alert').remove() + } + + $.support.transition && $parent.hasClass('fade') ? + $parent + .one($.support.transition.end, removeElement) + .emulateTransitionEnd(150) : + removeElement() + } + + + // ALERT PLUGIN DEFINITION + // ======================= + + var old = $.fn.alert + + $.fn.alert = function (option) { + return this.each(function () { + var $this = $(this) + var data = $this.data('bs.alert') + + if (!data) $this.data('bs.alert', (data = new Alert(this))) + if (typeof option == 'string') data[option].call($this) + }) + } + + $.fn.alert.Constructor = Alert + + + // ALERT NO CONFLICT + // ================= + + $.fn.alert.noConflict = function () { + $.fn.alert = old + return this + } + + + // ALERT DATA-API + // ============== + + $(document).on('click.bs.alert.data-api', dismiss, Alert.prototype.close) + +}(jQuery); + +/* ======================================================================== + * Bootstrap: button.js v3.1.1 + * http://getbootstrap.com/javascript/#buttons + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // BUTTON PUBLIC CLASS DEFINITION + // ============================== + + var Button = function (element, options) { + this.$element = $(element) + this.options = $.extend({}, Button.DEFAULTS, options) + this.isLoading = false + } + + Button.DEFAULTS = { + loadingText: 'loading...' + } + + Button.prototype.setState = function (state) { + var d = 'disabled' + var $el = this.$element + var val = $el.is('input') ? 'val' : 'html' + var data = $el.data() + + state = state + 'Text' + + if (!data.resetText) $el.data('resetText', $el[val]()) + + $el[val](data[state] || this.options[state]) + + // push to event loop to allow forms to submit + setTimeout($.proxy(function () { + if (state == 'loadingText') { + this.isLoading = true + $el.addClass(d).attr(d, d) + } else if (this.isLoading) { + this.isLoading = false + $el.removeClass(d).removeAttr(d) + } + }, this), 0) + } + + Button.prototype.toggle = function () { + var changed = true + var $parent = this.$element.closest('[data-toggle="buttons"]') + + if ($parent.length) { + var $input = this.$element.find('input') + if ($input.prop('type') == 'radio') { + if ($input.prop('checked') && this.$element.hasClass('active')) changed = false + else $parent.find('.active').removeClass('active') + } + if (changed) $input.prop('checked', !this.$element.hasClass('active')).trigger('change') + } + + if (changed) this.$element.toggleClass('active') + } + + + // BUTTON PLUGIN DEFINITION + // ======================== + + var old = $.fn.button + + $.fn.button = function (option) { + return this.each(function () { + var $this = $(this) + var data = $this.data('bs.button') + var options = typeof option == 'object' && option + + if (!data) $this.data('bs.button', (data = new Button(this, options))) + + if (option == 'toggle') data.toggle() + else if (option) data.setState(option) + }) + } + + $.fn.button.Constructor = Button + + + // BUTTON NO CONFLICT + // ================== + + $.fn.button.noConflict = function () { + $.fn.button = old + return this + } + + + // BUTTON DATA-API + // =============== + + $(document).on('click.bs.button.data-api', '[data-toggle^=button]', function (e) { + var $btn = $(e.target) + if (!$btn.hasClass('btn')) $btn = $btn.closest('.btn') + $btn.button('toggle') + e.preventDefault() + }) + +}(jQuery); + +/* ======================================================================== + * Bootstrap: carousel.js v3.1.1 + * http://getbootstrap.com/javascript/#carousel + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // CAROUSEL CLASS DEFINITION + // ========================= + + var Carousel = function (element, options) { + this.$element = $(element) + this.$indicators = this.$element.find('.carousel-indicators') + this.options = options + this.paused = + this.sliding = + this.interval = + this.$active = + this.$items = null + + this.options.pause == 'hover' && this.$element + .on('mouseenter', $.proxy(this.pause, this)) + .on('mouseleave', $.proxy(this.cycle, this)) + } + + Carousel.DEFAULTS = { + interval: 5000, + pause: 'hover', + wrap: true + } + + Carousel.prototype.cycle = function (e) { + e || (this.paused = false) + + this.interval && clearInterval(this.interval) + + this.options.interval + && !this.paused + && (this.interval = setInterval($.proxy(this.next, this), this.options.interval)) + + return this + } + + Carousel.prototype.getActiveIndex = function () { + this.$active = this.$element.find('.item.active') + this.$items = this.$active.parent().children() + + return this.$items.index(this.$active) + } + + Carousel.prototype.to = function (pos) { + var that = this + var activeIndex = this.getActiveIndex() + + if (pos > (this.$items.length - 1) || pos < 0) return + + if (this.sliding) return this.$element.one('slid.bs.carousel', function () { that.to(pos) }) + if (activeIndex == pos) return this.pause().cycle() + + return this.slide(pos > activeIndex ? 'next' : 'prev', $(this.$items[pos])) + } + + Carousel.prototype.pause = function (e) { + e || (this.paused = true) + + if (this.$element.find('.next, .prev').length && $.support.transition) { + this.$element.trigger($.support.transition.end) + this.cycle(true) + } + + this.interval = clearInterval(this.interval) + + return this + } + + Carousel.prototype.next = function () { + if (this.sliding) return + return this.slide('next') + } + + Carousel.prototype.prev = function () { + if (this.sliding) return + return this.slide('prev') + } + + Carousel.prototype.slide = function (type, next) { + var $active = this.$element.find('.item.active') + var $next = next || $active[type]() + var isCycling = this.interval + var direction = type == 'next' ? 'left' : 'right' + var fallback = type == 'next' ? 'first' : 'last' + var that = this + + if (!$next.length) { + if (!this.options.wrap) return + $next = this.$element.find('.item')[fallback]() + } + + if ($next.hasClass('active')) return this.sliding = false + + var e = $.Event('slide.bs.carousel', { relatedTarget: $next[0], direction: direction }) + this.$element.trigger(e) + if (e.isDefaultPrevented()) return + + this.sliding = true + + isCycling && this.pause() + + if (this.$indicators.length) { + this.$indicators.find('.active').removeClass('active') + this.$element.one('slid.bs.carousel', function () { + var $nextIndicator = $(that.$indicators.children()[that.getActiveIndex()]) + $nextIndicator && $nextIndicator.addClass('active') + }) + } + + if ($.support.transition && this.$element.hasClass('slide')) { + $next.addClass(type) + $next[0].offsetWidth // force reflow + $active.addClass(direction) + $next.addClass(direction) + $active + .one($.support.transition.end, function () { + $next.removeClass([type, direction].join(' ')).addClass('active') + $active.removeClass(['active', direction].join(' ')) + that.sliding = false + setTimeout(function () { that.$element.trigger('slid.bs.carousel') }, 0) + }) + .emulateTransitionEnd($active.css('transition-duration').slice(0, -1) * 1000) + } else { + $active.removeClass('active') + $next.addClass('active') + this.sliding = false + this.$element.trigger('slid.bs.carousel') + } + + isCycling && this.cycle() + + return this + } + + + // CAROUSEL PLUGIN DEFINITION + // ========================== + + var old = $.fn.carousel + + $.fn.carousel = function (option) { + return this.each(function () { + var $this = $(this) + var data = $this.data('bs.carousel') + var options = $.extend({}, Carousel.DEFAULTS, $this.data(), typeof option == 'object' && option) + var action = typeof option == 'string' ? option : options.slide + + if (!data) $this.data('bs.carousel', (data = new Carousel(this, options))) + if (typeof option == 'number') data.to(option) + else if (action) data[action]() + else if (options.interval) data.pause().cycle() + }) + } + + $.fn.carousel.Constructor = Carousel + + + // CAROUSEL NO CONFLICT + // ==================== + + $.fn.carousel.noConflict = function () { + $.fn.carousel = old + return this + } + + + // CAROUSEL DATA-API + // ================= + + $(document).on('click.bs.carousel.data-api', '[data-slide], [data-slide-to]', function (e) { + var $this = $(this), href + var $target = $($this.attr('data-target') || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7 + var options = $.extend({}, $target.data(), $this.data()) + var slideIndex = $this.attr('data-slide-to') + if (slideIndex) options.interval = false + + $target.carousel(options) + + if (slideIndex = $this.attr('data-slide-to')) { + $target.data('bs.carousel').to(slideIndex) + } + + e.preventDefault() + }) + + $(window).on('load', function () { + $('[data-ride="carousel"]').each(function () { + var $carousel = $(this) + $carousel.carousel($carousel.data()) + }) + }) + +}(jQuery); + +/* ======================================================================== + * Bootstrap: collapse.js v3.1.1 + * http://getbootstrap.com/javascript/#collapse + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // COLLAPSE PUBLIC CLASS DEFINITION + // ================================ + + var Collapse = function (element, options) { + this.$element = $(element) + this.options = $.extend({}, Collapse.DEFAULTS, options) + this.transitioning = null + + if (this.options.parent) this.$parent = $(this.options.parent) + if (this.options.toggle) this.toggle() + } + + Collapse.DEFAULTS = { + toggle: true + } + + Collapse.prototype.dimension = function () { + var hasWidth = this.$element.hasClass('width') + return hasWidth ? 'width' : 'height' + } + + Collapse.prototype.show = function () { + if (this.transitioning || this.$element.hasClass('in')) return + + var startEvent = $.Event('show.bs.collapse') + this.$element.trigger(startEvent) + if (startEvent.isDefaultPrevented()) return + + var actives = this.$parent && this.$parent.find('> .panel > .in') + + if (actives && actives.length) { + var hasData = actives.data('bs.collapse') + if (hasData && hasData.transitioning) return + actives.collapse('hide') + hasData || actives.data('bs.collapse', null) + } + + var dimension = this.dimension() + + this.$element + .removeClass('collapse') + .addClass('collapsing') + [dimension](0) + + this.transitioning = 1 + + var complete = function () { + this.$element + .removeClass('collapsing') + .addClass('collapse in') + [dimension]('auto') + this.transitioning = 0 + this.$element.trigger('shown.bs.collapse') + } + + if (!$.support.transition) return complete.call(this) + + var scrollSize = $.camelCase(['scroll', dimension].join('-')) + + this.$element + .one($.support.transition.end, $.proxy(complete, this)) + .emulateTransitionEnd(350) + [dimension](this.$element[0][scrollSize]) + } + + Collapse.prototype.hide = function () { + if (this.transitioning || !this.$element.hasClass('in')) return + + var startEvent = $.Event('hide.bs.collapse') + this.$element.trigger(startEvent) + if (startEvent.isDefaultPrevented()) return + + var dimension = this.dimension() + + this.$element + [dimension](this.$element[dimension]()) + [0].offsetHeight + + this.$element + .addClass('collapsing') + .removeClass('collapse') + .removeClass('in') + + this.transitioning = 1 + + var complete = function () { + this.transitioning = 0 + this.$element + .trigger('hidden.bs.collapse') + .removeClass('collapsing') + .addClass('collapse') + } + + if (!$.support.transition) return complete.call(this) + + this.$element + [dimension](0) + .one($.support.transition.end, $.proxy(complete, this)) + .emulateTransitionEnd(350) + } + + Collapse.prototype.toggle = function () { + this[this.$element.hasClass('in') ? 'hide' : 'show']() + } + + + // COLLAPSE PLUGIN DEFINITION + // ========================== + + var old = $.fn.collapse + + $.fn.collapse = function (option) { + return this.each(function () { + var $this = $(this) + var data = $this.data('bs.collapse') + var options = $.extend({}, Collapse.DEFAULTS, $this.data(), typeof option == 'object' && option) + + if (!data && options.toggle && option == 'show') option = !option + if (!data) $this.data('bs.collapse', (data = new Collapse(this, options))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.collapse.Constructor = Collapse + + + // COLLAPSE NO CONFLICT + // ==================== + + $.fn.collapse.noConflict = function () { + $.fn.collapse = old + return this + } + + + // COLLAPSE DATA-API + // ================= + + $(document).on('click.bs.collapse.data-api', '[data-toggle=collapse]', function (e) { + var $this = $(this), href + var target = $this.attr('data-target') + || e.preventDefault() + || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '') //strip for ie7 + var $target = $(target) + var data = $target.data('bs.collapse') + var option = data ? 'toggle' : $this.data() + var parent = $this.attr('data-parent') + var $parent = parent && $(parent) + + if (!data || !data.transitioning) { + if ($parent) $parent.find('[data-toggle=collapse][data-parent="' + parent + '"]').not($this).addClass('collapsed') + $this[$target.hasClass('in') ? 'addClass' : 'removeClass']('collapsed') + } + + $target.collapse(option) + }) + +}(jQuery); + +/* ======================================================================== + * Bootstrap: dropdown.js v3.1.1 + * http://getbootstrap.com/javascript/#dropdowns + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // DROPDOWN CLASS DEFINITION + // ========================= + + var backdrop = '.dropdown-backdrop' + var toggle = '[data-toggle=dropdown]' + var Dropdown = function (element) { + $(element).on('click.bs.dropdown', this.toggle) + } + + Dropdown.prototype.toggle = function (e) { + var $this = $(this) + + if ($this.is('.disabled, :disabled')) return + + var $parent = getParent($this) + var isActive = $parent.hasClass('open') + + clearMenus() + + if (!isActive) { + if ('ontouchstart' in document.documentElement && !$parent.closest('.navbar-nav').length) { + // if mobile we use a backdrop because click events don't delegate + $('
    ').insertAfter($(this)).on('click', clearMenus) + } + + var relatedTarget = { relatedTarget: this } + $parent.trigger(e = $.Event('show.bs.dropdown', relatedTarget)) + + if (e.isDefaultPrevented()) return + + $parent + .toggleClass('open') + .trigger('shown.bs.dropdown', relatedTarget) + + $this.focus() + } + + return false + } + + Dropdown.prototype.keydown = function (e) { + if (!/(38|40|27)/.test(e.keyCode)) return + + var $this = $(this) + + e.preventDefault() + e.stopPropagation() + + if ($this.is('.disabled, :disabled')) return + + var $parent = getParent($this) + var isActive = $parent.hasClass('open') + + if (!isActive || (isActive && e.keyCode == 27)) { + if (e.which == 27) $parent.find(toggle).focus() + return $this.click() + } + + var desc = ' li:not(.divider):visible a' + var $items = $parent.find('[role=menu]' + desc + ', [role=listbox]' + desc) + + if (!$items.length) return + + var index = $items.index($items.filter(':focus')) + + if (e.keyCode == 38 && index > 0) index-- // up + if (e.keyCode == 40 && index < $items.length - 1) index++ // down + if (!~index) index = 0 + + $items.eq(index).focus() + } + + function clearMenus(e) { + $(backdrop).remove() + $(toggle).each(function () { + var $parent = getParent($(this)) + var relatedTarget = { relatedTarget: this } + if (!$parent.hasClass('open')) return + $parent.trigger(e = $.Event('hide.bs.dropdown', relatedTarget)) + if (e.isDefaultPrevented()) return + $parent.removeClass('open').trigger('hidden.bs.dropdown', relatedTarget) + }) + } + + function getParent($this) { + var selector = $this.attr('data-target') + + if (!selector) { + selector = $this.attr('href') + selector = selector && /#[A-Za-z]/.test(selector) && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 + } + + var $parent = selector && $(selector) + + return $parent && $parent.length ? $parent : $this.parent() + } + + + // DROPDOWN PLUGIN DEFINITION + // ========================== + + var old = $.fn.dropdown + + $.fn.dropdown = function (option) { + return this.each(function () { + var $this = $(this) + var data = $this.data('bs.dropdown') + + if (!data) $this.data('bs.dropdown', (data = new Dropdown(this))) + if (typeof option == 'string') data[option].call($this) + }) + } + + $.fn.dropdown.Constructor = Dropdown + + + // DROPDOWN NO CONFLICT + // ==================== + + $.fn.dropdown.noConflict = function () { + $.fn.dropdown = old + return this + } + + + // APPLY TO STANDARD DROPDOWN ELEMENTS + // =================================== + + $(document) + .on('click.bs.dropdown.data-api', clearMenus) + .on('click.bs.dropdown.data-api', '.dropdown form', function (e) { e.stopPropagation() }) + .on('click.bs.dropdown.data-api', toggle, Dropdown.prototype.toggle) + .on('keydown.bs.dropdown.data-api', toggle + ', [role=menu], [role=listbox]', Dropdown.prototype.keydown) + +}(jQuery); + +/* ======================================================================== + * Bootstrap: modal.js v3.1.1 + * http://getbootstrap.com/javascript/#modals + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // MODAL CLASS DEFINITION + // ====================== + + var Modal = function (element, options) { + this.options = options + this.$element = $(element) + this.$backdrop = + this.isShown = null + + if (this.options.remote) { + this.$element + .find('.modal-content') + .load(this.options.remote, $.proxy(function () { + this.$element.trigger('loaded.bs.modal') + }, this)) + } + } + + Modal.DEFAULTS = { + backdrop: true, + keyboard: true, + show: true + } + + Modal.prototype.toggle = function (_relatedTarget) { + return this[!this.isShown ? 'show' : 'hide'](_relatedTarget) + } + + Modal.prototype.show = function (_relatedTarget) { + var that = this + var e = $.Event('show.bs.modal', { relatedTarget: _relatedTarget }) + + this.$element.trigger(e) + + if (this.isShown || e.isDefaultPrevented()) return + + this.isShown = true + + this.escape() + + this.$element.on('click.dismiss.bs.modal', '[data-dismiss="modal"]', $.proxy(this.hide, this)) + + this.backdrop(function () { + var transition = $.support.transition && that.$element.hasClass('fade') + + if (!that.$element.parent().length) { + that.$element.appendTo(document.body) // don't move modals dom position + } + + that.$element + .show() + .scrollTop(0) + + if (transition) { + that.$element[0].offsetWidth // force reflow + } + + that.$element + .addClass('in') + .attr('aria-hidden', false) + + that.enforceFocus() + + var e = $.Event('shown.bs.modal', { relatedTarget: _relatedTarget }) + + transition ? + that.$element.find('.modal-dialog') // wait for modal to slide in + .one($.support.transition.end, function () { + that.$element.focus().trigger(e) + }) + .emulateTransitionEnd(300) : + that.$element.focus().trigger(e) + }) + } + + Modal.prototype.hide = function (e) { + if (e) e.preventDefault() + + e = $.Event('hide.bs.modal') + + this.$element.trigger(e) + + if (!this.isShown || e.isDefaultPrevented()) return + + this.isShown = false + + this.escape() + + $(document).off('focusin.bs.modal') + + this.$element + .removeClass('in') + .attr('aria-hidden', true) + .off('click.dismiss.bs.modal') + + $.support.transition && this.$element.hasClass('fade') ? + this.$element + .one($.support.transition.end, $.proxy(this.hideModal, this)) + .emulateTransitionEnd(300) : + this.hideModal() + } + + Modal.prototype.enforceFocus = function () { + $(document) + .off('focusin.bs.modal') // guard against infinite focus loop + .on('focusin.bs.modal', $.proxy(function (e) { + if (this.$element[0] !== e.target && !this.$element.has(e.target).length) { + this.$element.focus() + } + }, this)) + } + + Modal.prototype.escape = function () { + if (this.isShown && this.options.keyboard) { + this.$element.on('keyup.dismiss.bs.modal', $.proxy(function (e) { + e.which == 27 && this.hide() + }, this)) + } else if (!this.isShown) { + this.$element.off('keyup.dismiss.bs.modal') + } + } + + Modal.prototype.hideModal = function () { + var that = this + this.$element.hide() + this.backdrop(function () { + that.removeBackdrop() + that.$element.trigger('hidden.bs.modal') + }) + } + + Modal.prototype.removeBackdrop = function () { + this.$backdrop && this.$backdrop.remove() + this.$backdrop = null + } + + Modal.prototype.backdrop = function (callback) { + var animate = this.$element.hasClass('fade') ? 'fade' : '' + + if (this.isShown && this.options.backdrop) { + var doAnimate = $.support.transition && animate + + this.$backdrop = $('
    ') + .appendTo(document.body) + + this.$element.on('click.dismiss.bs.modal', $.proxy(function (e) { + if (e.target !== e.currentTarget) return + this.options.backdrop == 'static' + ? this.$element[0].focus.call(this.$element[0]) + : this.hide.call(this) + }, this)) + + if (doAnimate) this.$backdrop[0].offsetWidth // force reflow + + this.$backdrop.addClass('in') + + if (!callback) return + + doAnimate ? + this.$backdrop + .one($.support.transition.end, callback) + .emulateTransitionEnd(150) : + callback() + + } else if (!this.isShown && this.$backdrop) { + this.$backdrop.removeClass('in') + + $.support.transition && this.$element.hasClass('fade') ? + this.$backdrop + .one($.support.transition.end, callback) + .emulateTransitionEnd(150) : + callback() + + } else if (callback) { + callback() + } + } + + + // MODAL PLUGIN DEFINITION + // ======================= + + var old = $.fn.modal + + $.fn.modal = function (option, _relatedTarget) { + return this.each(function () { + var $this = $(this) + var data = $this.data('bs.modal') + var options = $.extend({}, Modal.DEFAULTS, $this.data(), typeof option == 'object' && option) + + if (!data) $this.data('bs.modal', (data = new Modal(this, options))) + if (typeof option == 'string') data[option](_relatedTarget) + else if (options.show) data.show(_relatedTarget) + }) + } + + $.fn.modal.Constructor = Modal + + + // MODAL NO CONFLICT + // ================= + + $.fn.modal.noConflict = function () { + $.fn.modal = old + return this + } + + + // MODAL DATA-API + // ============== + + $(document).on('click.bs.modal.data-api', '[data-toggle="modal"]', function (e) { + var $this = $(this) + var href = $this.attr('href') + var $target = $($this.attr('data-target') || (href && href.replace(/.*(?=#[^\s]+$)/, ''))) //strip for ie7 + var option = $target.data('bs.modal') ? 'toggle' : $.extend({ remote: !/#/.test(href) && href }, $target.data(), $this.data()) + + if ($this.is('a')) e.preventDefault() + + $target + .modal(option, this) + .one('hide', function () { + $this.is(':visible') && $this.focus() + }) + }) + + $(document) + .on('show.bs.modal', '.modal', function () { $(document.body).addClass('modal-open') }) + .on('hidden.bs.modal', '.modal', function () { $(document.body).removeClass('modal-open') }) + +}(jQuery); + +/* ======================================================================== + * Bootstrap: tooltip.js v3.1.1 + * http://getbootstrap.com/javascript/#tooltip + * Inspired by the original jQuery.tipsy by Jason Frame + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // TOOLTIP PUBLIC CLASS DEFINITION + // =============================== + + var Tooltip = function (element, options) { + this.type = + this.options = + this.enabled = + this.timeout = + this.hoverState = + this.$element = null + + this.init('tooltip', element, options) + } + + Tooltip.DEFAULTS = { + animation: true, + placement: 'top', + selector: false, + template: '
    ', + trigger: 'hover focus', + title: '', + delay: 0, + html: false, + container: false + } + + Tooltip.prototype.init = function (type, element, options) { + this.enabled = true + this.type = type + this.$element = $(element) + this.options = this.getOptions(options) + + var triggers = this.options.trigger.split(' ') + + for (var i = triggers.length; i--;) { + var trigger = triggers[i] + + if (trigger == 'click') { + this.$element.on('click.' + this.type, this.options.selector, $.proxy(this.toggle, this)) + } else if (trigger != 'manual') { + var eventIn = trigger == 'hover' ? 'mouseenter' : 'focusin' + var eventOut = trigger == 'hover' ? 'mouseleave' : 'focusout' + + this.$element.on(eventIn + '.' + this.type, this.options.selector, $.proxy(this.enter, this)) + this.$element.on(eventOut + '.' + this.type, this.options.selector, $.proxy(this.leave, this)) + } + } + + this.options.selector ? + (this._options = $.extend({}, this.options, { trigger: 'manual', selector: '' })) : + this.fixTitle() + } + + Tooltip.prototype.getDefaults = function () { + return Tooltip.DEFAULTS + } + + Tooltip.prototype.getOptions = function (options) { + options = $.extend({}, this.getDefaults(), this.$element.data(), options) + + if (options.delay && typeof options.delay == 'number') { + options.delay = { + show: options.delay, + hide: options.delay + } + } + + return options + } + + Tooltip.prototype.getDelegateOptions = function () { + var options = {} + var defaults = this.getDefaults() + + this._options && $.each(this._options, function (key, value) { + if (defaults[key] != value) options[key] = value + }) + + return options + } + + Tooltip.prototype.enter = function (obj) { + var self = obj instanceof this.constructor ? + obj : $(obj.currentTarget)[this.type](this.getDelegateOptions()).data('bs.' + this.type) + + clearTimeout(self.timeout) + + self.hoverState = 'in' + + if (!self.options.delay || !self.options.delay.show) return self.show() + + self.timeout = setTimeout(function () { + if (self.hoverState == 'in') self.show() + }, self.options.delay.show) + } + + Tooltip.prototype.leave = function (obj) { + var self = obj instanceof this.constructor ? + obj : $(obj.currentTarget)[this.type](this.getDelegateOptions()).data('bs.' + this.type) + + clearTimeout(self.timeout) + + self.hoverState = 'out' + + if (!self.options.delay || !self.options.delay.hide) return self.hide() + + self.timeout = setTimeout(function () { + if (self.hoverState == 'out') self.hide() + }, self.options.delay.hide) + } + + Tooltip.prototype.show = function () { + var e = $.Event('show.bs.' + this.type) + + if (this.hasContent() && this.enabled) { + this.$element.trigger(e) + + if (e.isDefaultPrevented()) return + var that = this; + + var $tip = this.tip() + + this.setContent() + + if (this.options.animation) $tip.addClass('fade') + + var placement = typeof this.options.placement == 'function' ? + this.options.placement.call(this, $tip[0], this.$element[0]) : + this.options.placement + + var autoToken = /\s?auto?\s?/i + var autoPlace = autoToken.test(placement) + if (autoPlace) placement = placement.replace(autoToken, '') || 'top' + + $tip + .detach() + .css({ top: 0, left: 0, display: 'block' }) + .addClass(placement) + + this.options.container ? $tip.appendTo(this.options.container) : $tip.insertAfter(this.$element) + + var pos = this.getPosition() + var actualWidth = $tip[0].offsetWidth + var actualHeight = $tip[0].offsetHeight + + if (autoPlace) { + var $parent = this.$element.parent() + + var orgPlacement = placement + var docScroll = document.documentElement.scrollTop || document.body.scrollTop + var parentWidth = this.options.container == 'body' ? window.innerWidth : $parent.outerWidth() + var parentHeight = this.options.container == 'body' ? window.innerHeight : $parent.outerHeight() + var parentLeft = this.options.container == 'body' ? 0 : $parent.offset().left + + placement = placement == 'bottom' && pos.top + pos.height + actualHeight - docScroll > parentHeight ? 'top' : + placement == 'top' && pos.top - docScroll - actualHeight < 0 ? 'bottom' : + placement == 'right' && pos.right + actualWidth > parentWidth ? 'left' : + placement == 'left' && pos.left - actualWidth < parentLeft ? 'right' : + placement + + $tip + .removeClass(orgPlacement) + .addClass(placement) + } + + var calculatedOffset = this.getCalculatedOffset(placement, pos, actualWidth, actualHeight) + + this.applyPlacement(calculatedOffset, placement) + this.hoverState = null + + var complete = function() { + that.$element.trigger('shown.bs.' + that.type) + } + + $.support.transition && this.$tip.hasClass('fade') ? + $tip + .one($.support.transition.end, complete) + .emulateTransitionEnd(150) : + complete() + } + } + + Tooltip.prototype.applyPlacement = function (offset, placement) { + var replace + var $tip = this.tip() + var width = $tip[0].offsetWidth + var height = $tip[0].offsetHeight + + // manually read margins because getBoundingClientRect includes difference + var marginTop = parseInt($tip.css('margin-top'), 10) + var marginLeft = parseInt($tip.css('margin-left'), 10) + + // we must check for NaN for ie 8/9 + if (isNaN(marginTop)) marginTop = 0 + if (isNaN(marginLeft)) marginLeft = 0 + + offset.top = offset.top + marginTop + offset.left = offset.left + marginLeft + + // $.fn.offset doesn't round pixel values + // so we use setOffset directly with our own function B-0 + $.offset.setOffset($tip[0], $.extend({ + using: function (props) { + $tip.css({ + top: Math.round(props.top), + left: Math.round(props.left) + }) + } + }, offset), 0) + + $tip.addClass('in') + + // check to see if placing tip in new offset caused the tip to resize itself + var actualWidth = $tip[0].offsetWidth + var actualHeight = $tip[0].offsetHeight + + if (placement == 'top' && actualHeight != height) { + replace = true + offset.top = offset.top + height - actualHeight + } + + if (/bottom|top/.test(placement)) { + var delta = 0 + + if (offset.left < 0) { + delta = offset.left * -2 + offset.left = 0 + + $tip.offset(offset) + + actualWidth = $tip[0].offsetWidth + actualHeight = $tip[0].offsetHeight + } + + this.replaceArrow(delta - width + actualWidth, actualWidth, 'left') + } else { + this.replaceArrow(actualHeight - height, actualHeight, 'top') + } + + if (replace) $tip.offset(offset) + } + + Tooltip.prototype.replaceArrow = function (delta, dimension, position) { + this.arrow().css(position, delta ? (50 * (1 - delta / dimension) + '%') : '') + } + + Tooltip.prototype.setContent = function () { + var $tip = this.tip() + var title = this.getTitle() + + $tip.find('.tooltip-inner')[this.options.html ? 'html' : 'text'](title) + $tip.removeClass('fade in top bottom left right') + } + + Tooltip.prototype.hide = function () { + var that = this + var $tip = this.tip() + var e = $.Event('hide.bs.' + this.type) + + function complete() { + if (that.hoverState != 'in') $tip.detach() + that.$element.trigger('hidden.bs.' + that.type) + } + + this.$element.trigger(e) + + if (e.isDefaultPrevented()) return + + $tip.removeClass('in') + + $.support.transition && this.$tip.hasClass('fade') ? + $tip + .one($.support.transition.end, complete) + .emulateTransitionEnd(150) : + complete() + + this.hoverState = null + + return this + } + + Tooltip.prototype.fixTitle = function () { + var $e = this.$element + if ($e.attr('title') || typeof($e.attr('data-original-title')) != 'string') { + $e.attr('data-original-title', $e.attr('title') || '').attr('title', '') + } + } + + Tooltip.prototype.hasContent = function () { + return this.getTitle() + } + + Tooltip.prototype.getPosition = function () { + var el = this.$element[0] + return $.extend({}, (typeof el.getBoundingClientRect == 'function') ? el.getBoundingClientRect() : { + width: el.offsetWidth, + height: el.offsetHeight + }, this.$element.offset()) + } + + Tooltip.prototype.getCalculatedOffset = function (placement, pos, actualWidth, actualHeight) { + return placement == 'bottom' ? { top: pos.top + pos.height, left: pos.left + pos.width / 2 - actualWidth / 2 } : + placement == 'top' ? { top: pos.top - actualHeight, left: pos.left + pos.width / 2 - actualWidth / 2 } : + placement == 'left' ? { top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left - actualWidth } : + /* placement == 'right' */ { top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left + pos.width } + } + + Tooltip.prototype.getTitle = function () { + var title + var $e = this.$element + var o = this.options + + title = $e.attr('data-original-title') + || (typeof o.title == 'function' ? o.title.call($e[0]) : o.title) + + return title + } + + Tooltip.prototype.tip = function () { + return this.$tip = this.$tip || $(this.options.template) + } + + Tooltip.prototype.arrow = function () { + return this.$arrow = this.$arrow || this.tip().find('.tooltip-arrow') + } + + Tooltip.prototype.validate = function () { + if (!this.$element[0].parentNode) { + this.hide() + this.$element = null + this.options = null + } + } + + Tooltip.prototype.enable = function () { + this.enabled = true + } + + Tooltip.prototype.disable = function () { + this.enabled = false + } + + Tooltip.prototype.toggleEnabled = function () { + this.enabled = !this.enabled + } + + Tooltip.prototype.toggle = function (e) { + var self = e ? $(e.currentTarget)[this.type](this.getDelegateOptions()).data('bs.' + this.type) : this + self.tip().hasClass('in') ? self.leave(self) : self.enter(self) + } + + Tooltip.prototype.destroy = function () { + clearTimeout(this.timeout) + this.hide().$element.off('.' + this.type).removeData('bs.' + this.type) + } + + + // TOOLTIP PLUGIN DEFINITION + // ========================= + + var old = $.fn.tooltip + + $.fn.tooltip = function (option) { + return this.each(function () { + var $this = $(this) + var data = $this.data('bs.tooltip') + var options = typeof option == 'object' && option + + if (!data && option == 'destroy') return + if (!data) $this.data('bs.tooltip', (data = new Tooltip(this, options))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.tooltip.Constructor = Tooltip + + + // TOOLTIP NO CONFLICT + // =================== + + $.fn.tooltip.noConflict = function () { + $.fn.tooltip = old + return this + } + +}(jQuery); + +/* ======================================================================== + * Bootstrap: popover.js v3.1.1 + * http://getbootstrap.com/javascript/#popovers + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // POPOVER PUBLIC CLASS DEFINITION + // =============================== + + var Popover = function (element, options) { + this.init('popover', element, options) + } + + if (!$.fn.tooltip) throw new Error('Popover requires tooltip.js') + + Popover.DEFAULTS = $.extend({}, $.fn.tooltip.Constructor.DEFAULTS, { + placement: 'right', + trigger: 'click', + content: '', + template: '

    ' + }) + + + // NOTE: POPOVER EXTENDS tooltip.js + // ================================ + + Popover.prototype = $.extend({}, $.fn.tooltip.Constructor.prototype) + + Popover.prototype.constructor = Popover + + Popover.prototype.getDefaults = function () { + return Popover.DEFAULTS + } + + Popover.prototype.setContent = function () { + var $tip = this.tip() + var title = this.getTitle() + var content = this.getContent() + + $tip.find('.popover-title')[this.options.html ? 'html' : 'text'](title) + $tip.find('.popover-content')[ // we use append for html objects to maintain js events + this.options.html ? (typeof content == 'string' ? 'html' : 'append') : 'text' + ](content) + + $tip.removeClass('fade top bottom left right in') + + // IE8 doesn't accept hiding via the `:empty` pseudo selector, we have to do + // this manually by checking the contents. + if (!$tip.find('.popover-title').html()) $tip.find('.popover-title').hide() + } + + Popover.prototype.hasContent = function () { + return this.getTitle() || this.getContent() + } + + Popover.prototype.getContent = function () { + var $e = this.$element + var o = this.options + + return $e.attr('data-content') + || (typeof o.content == 'function' ? + o.content.call($e[0]) : + o.content) + } + + Popover.prototype.arrow = function () { + return this.$arrow = this.$arrow || this.tip().find('.arrow') + } + + Popover.prototype.tip = function () { + if (!this.$tip) this.$tip = $(this.options.template) + return this.$tip + } + + + // POPOVER PLUGIN DEFINITION + // ========================= + + var old = $.fn.popover + + $.fn.popover = function (option) { + return this.each(function () { + var $this = $(this) + var data = $this.data('bs.popover') + var options = typeof option == 'object' && option + + if (!data && option == 'destroy') return + if (!data) $this.data('bs.popover', (data = new Popover(this, options))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.popover.Constructor = Popover + + + // POPOVER NO CONFLICT + // =================== + + $.fn.popover.noConflict = function () { + $.fn.popover = old + return this + } + +}(jQuery); + +/* ======================================================================== + * Bootstrap: scrollspy.js v3.1.1 + * http://getbootstrap.com/javascript/#scrollspy + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // SCROLLSPY CLASS DEFINITION + // ========================== + + function ScrollSpy(element, options) { + var href + var process = $.proxy(this.process, this) + + this.$element = $(element).is('body') ? $(window) : $(element) + this.$body = $('body') + this.$scrollElement = this.$element.on('scroll.bs.scroll-spy.data-api', process) + this.options = $.extend({}, ScrollSpy.DEFAULTS, options) + this.selector = (this.options.target + || ((href = $(element).attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7 + || '') + ' .nav li > a' + this.offsets = $([]) + this.targets = $([]) + this.activeTarget = null + + this.refresh() + this.process() + } + + ScrollSpy.DEFAULTS = { + offset: 10 + } + + ScrollSpy.prototype.refresh = function () { + var offsetMethod = this.$element[0] == window ? 'offset' : 'position' + + this.offsets = $([]) + this.targets = $([]) + + var self = this + var $targets = this.$body + .find(this.selector) + .map(function () { + var $el = $(this) + var href = $el.data('target') || $el.attr('href') + var $href = /^#./.test(href) && $(href) + + return ($href + && $href.length + && $href.is(':visible') + && [[ $href[offsetMethod]().top + (!$.isWindow(self.$scrollElement.get(0)) && self.$scrollElement.scrollTop()), href ]]) || null + }) + .sort(function (a, b) { return a[0] - b[0] }) + .each(function () { + self.offsets.push(this[0]) + self.targets.push(this[1]) + }) + } + + ScrollSpy.prototype.process = function () { + var scrollTop = this.$scrollElement.scrollTop() + this.options.offset + var scrollHeight = this.$scrollElement[0].scrollHeight || this.$body[0].scrollHeight + var maxScroll = scrollHeight - this.$scrollElement.height() + var offsets = this.offsets + var targets = this.targets + var activeTarget = this.activeTarget + var i + + if (scrollTop >= maxScroll) { + return activeTarget != (i = targets.last()[0]) && this.activate(i) + } + + if (activeTarget && scrollTop <= offsets[0]) { + return activeTarget != (i = targets[0]) && this.activate(i) + } + + for (i = offsets.length; i--;) { + activeTarget != targets[i] + && scrollTop >= offsets[i] + && (!offsets[i + 1] || scrollTop <= offsets[i + 1]) + && this.activate( targets[i] ) + } + } + + ScrollSpy.prototype.activate = function (target) { + this.activeTarget = target + + $(this.selector) + .parentsUntil(this.options.target, '.active') + .removeClass('active') + + var selector = this.selector + + '[data-target="' + target + '"],' + + this.selector + '[href="' + target + '"]' + + var active = $(selector) + .parents('li') + .addClass('active') + + if (active.parent('.dropdown-menu').length) { + active = active + .closest('li.dropdown') + .addClass('active') + } + + active.trigger('activate.bs.scrollspy') + } + + + // SCROLLSPY PLUGIN DEFINITION + // =========================== + + var old = $.fn.scrollspy + + $.fn.scrollspy = function (option) { + return this.each(function () { + var $this = $(this) + var data = $this.data('bs.scrollspy') + var options = typeof option == 'object' && option + + if (!data) $this.data('bs.scrollspy', (data = new ScrollSpy(this, options))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.scrollspy.Constructor = ScrollSpy + + + // SCROLLSPY NO CONFLICT + // ===================== + + $.fn.scrollspy.noConflict = function () { + $.fn.scrollspy = old + return this + } + + + // SCROLLSPY DATA-API + // ================== + + $(window).on('load', function () { + $('[data-spy="scroll"]').each(function () { + var $spy = $(this) + $spy.scrollspy($spy.data()) + }) + }) + +}(jQuery); + +/* ======================================================================== + * Bootstrap: tab.js v3.1.1 + * http://getbootstrap.com/javascript/#tabs + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // TAB CLASS DEFINITION + // ==================== + + var Tab = function (element) { + this.element = $(element) + } + + Tab.prototype.show = function () { + var $this = this.element + var $ul = $this.closest('ul:not(.dropdown-menu)') + var selector = $this.data('target') + + if (!selector) { + selector = $this.attr('href') + selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 + } + + if ($this.parent('li').hasClass('active')) return + + var previous = $ul.find('.active:last a')[0] + var e = $.Event('show.bs.tab', { + relatedTarget: previous + }) + + $this.trigger(e) + + if (e.isDefaultPrevented()) return + + var $target = $(selector) + + this.activate($this.parent('li'), $ul) + this.activate($target, $target.parent(), function () { + $this.trigger({ + type: 'shown.bs.tab', + relatedTarget: previous + }) + }) + } + + Tab.prototype.activate = function (element, container, callback) { + var $active = container.find('> .active') + var transition = callback + && $.support.transition + && $active.hasClass('fade') + + function next() { + $active + .removeClass('active') + .find('> .dropdown-menu > .active') + .removeClass('active') + + element.addClass('active') + + if (transition) { + element[0].offsetWidth // reflow for transition + element.addClass('in') + } else { + element.removeClass('fade') + } + + if (element.parent('.dropdown-menu')) { + element.closest('li.dropdown').addClass('active') + } + + callback && callback() + } + + transition ? + $active + .one($.support.transition.end, next) + .emulateTransitionEnd(150) : + next() + + $active.removeClass('in') + } + + + // TAB PLUGIN DEFINITION + // ===================== + + var old = $.fn.tab + + $.fn.tab = function ( option ) { + return this.each(function () { + var $this = $(this) + var data = $this.data('bs.tab') + + if (!data) $this.data('bs.tab', (data = new Tab(this))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.tab.Constructor = Tab + + + // TAB NO CONFLICT + // =============== + + $.fn.tab.noConflict = function () { + $.fn.tab = old + return this + } + + + // TAB DATA-API + // ============ + + $(document).on('click.bs.tab.data-api', '[data-toggle="tab"], [data-toggle="pill"]', function (e) { + e.preventDefault() + $(this).tab('show') + }) + +}(jQuery); + +/* ======================================================================== + * Bootstrap: affix.js v3.1.1 + * http://getbootstrap.com/javascript/#affix + * ======================================================================== + * Copyright 2011-2014 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * ======================================================================== */ + + ++function ($) { + 'use strict'; + + // AFFIX CLASS DEFINITION + // ====================== + + var Affix = function (element, options) { + this.options = $.extend({}, Affix.DEFAULTS, options) + this.$window = $(window) + .on('scroll.bs.affix.data-api', $.proxy(this.checkPosition, this)) + .on('click.bs.affix.data-api', $.proxy(this.checkPositionWithEventLoop, this)) + + this.$element = $(element) + this.affixed = + this.unpin = + this.pinnedOffset = null + + this.checkPosition() + } + + Affix.RESET = 'affix affix-top affix-bottom' + + Affix.DEFAULTS = { + offset: 0 + } + + Affix.prototype.getPinnedOffset = function () { + if (this.pinnedOffset) return this.pinnedOffset + this.$element.removeClass(Affix.RESET).addClass('affix') + var scrollTop = this.$window.scrollTop() + var position = this.$element.offset() + return (this.pinnedOffset = position.top - scrollTop) + } + + Affix.prototype.checkPositionWithEventLoop = function () { + setTimeout($.proxy(this.checkPosition, this), 1) + } + + Affix.prototype.checkPosition = function () { + if (!this.$element.is(':visible')) return + + var scrollHeight = $(document).height() + var scrollTop = this.$window.scrollTop() + var position = this.$element.offset() + var offset = this.options.offset + var offsetTop = offset.top + var offsetBottom = offset.bottom + + if (this.affixed == 'top') position.top += scrollTop + + if (typeof offset != 'object') offsetBottom = offsetTop = offset + if (typeof offsetTop == 'function') offsetTop = offset.top(this.$element) + if (typeof offsetBottom == 'function') offsetBottom = offset.bottom(this.$element) + + var affix = this.unpin != null && (scrollTop + this.unpin <= position.top) ? false : + offsetBottom != null && (position.top + this.$element.height() >= scrollHeight - offsetBottom) ? 'bottom' : + offsetTop != null && (scrollTop <= offsetTop) ? 'top' : false + + if (this.affixed === affix) return + if (this.unpin) this.$element.css('top', '') + + var affixType = 'affix' + (affix ? '-' + affix : '') + var e = $.Event(affixType + '.bs.affix') + + this.$element.trigger(e) + + if (e.isDefaultPrevented()) return + + this.affixed = affix + this.unpin = affix == 'bottom' ? this.getPinnedOffset() : null + + this.$element + .removeClass(Affix.RESET) + .addClass(affixType) + .trigger($.Event(affixType.replace('affix', 'affixed'))) + + if (affix == 'bottom') { + this.$element.offset({ top: scrollHeight - offsetBottom - this.$element.height() }) + } + } + + + // AFFIX PLUGIN DEFINITION + // ======================= + + var old = $.fn.affix + + $.fn.affix = function (option) { + return this.each(function () { + var $this = $(this) + var data = $this.data('bs.affix') + var options = typeof option == 'object' && option + + if (!data) $this.data('bs.affix', (data = new Affix(this, options))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.affix.Constructor = Affix + + + // AFFIX NO CONFLICT + // ================= + + $.fn.affix.noConflict = function () { + $.fn.affix = old + return this + } + + + // AFFIX DATA-API + // ============== + + $(window).on('load', function () { + $('[data-spy="affix"]').each(function () { + var $spy = $(this) + var data = $spy.data() + + data.offset = data.offset || {} + + if (data.offsetBottom) data.offset.bottom = data.offsetBottom + if (data.offsetTop) data.offset.top = data.offsetTop + + $spy.affix(data) + }) + }) + +}(jQuery); diff --git a/.rclone_repo/docs/static/js/custom.js b/.rclone_repo/docs/static/js/custom.js new file mode 100755 index 0000000..6638768 --- /dev/null +++ b/.rclone_repo/docs/static/js/custom.js @@ -0,0 +1,14 @@ +// Site JS + +// Add hover links on headings +$(function() { + return $("h2, h3, h4, h5, h6").each(function(i, el) { + var $el, icon, id; + $el = $(el); + id = $el.attr('id'); + icon = ''; + if (id) { + return $el.prepend($("").addClass("header-link").attr("href", "#" + id).html(icon)); + } + }); +}); diff --git a/.rclone_repo/docs/static/js/jquery.js b/.rclone_repo/docs/static/js/jquery.js new file mode 100755 index 0000000..3c88fa8 --- /dev/null +++ b/.rclone_repo/docs/static/js/jquery.js @@ -0,0 +1,10337 @@ +/*! + * jQuery JavaScript Library v1.11.0 + * http://jquery.com/ + * + * Includes Sizzle.js + * http://sizzlejs.com/ + * + * Copyright 2005, 2014 jQuery Foundation, Inc. and other contributors + * Released under the MIT license + * http://jquery.org/license + * + * Date: 2014-01-23T21:02Z + */ + +(function( global, factory ) { + + if ( typeof module === "object" && typeof module.exports === "object" ) { + // For CommonJS and CommonJS-like environments where a proper window is present, + // execute the factory and get jQuery + // For environments that do not inherently posses a window with a document + // (such as Node.js), expose a jQuery-making factory as module.exports + // This accentuates the need for the creation of a real window + // e.g. var jQuery = require("jquery")(window); + // See ticket #14549 for more info + module.exports = global.document ? + factory( global, true ) : + function( w ) { + if ( !w.document ) { + throw new Error( "jQuery requires a window with a document" ); + } + return factory( w ); + }; + } else { + factory( global ); + } + +// Pass this if window is not defined yet +}(typeof window !== "undefined" ? window : this, function( window, noGlobal ) { + +// Can't do this because several apps including ASP.NET trace +// the stack via arguments.caller.callee and Firefox dies if +// you try to trace through "use strict" call chains. (#13335) +// Support: Firefox 18+ +// + +var deletedIds = []; + +var slice = deletedIds.slice; + +var concat = deletedIds.concat; + +var push = deletedIds.push; + +var indexOf = deletedIds.indexOf; + +var class2type = {}; + +var toString = class2type.toString; + +var hasOwn = class2type.hasOwnProperty; + +var trim = "".trim; + +var support = {}; + + + +var + version = "1.11.0", + + // Define a local copy of jQuery + jQuery = function( selector, context ) { + // The jQuery object is actually just the init constructor 'enhanced' + // Need init if jQuery is called (just allow error to be thrown if not included) + return new jQuery.fn.init( selector, context ); + }, + + // Make sure we trim BOM and NBSP (here's looking at you, Safari 5.0 and IE) + rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g, + + // Matches dashed string for camelizing + rmsPrefix = /^-ms-/, + rdashAlpha = /-([\da-z])/gi, + + // Used by jQuery.camelCase as callback to replace() + fcamelCase = function( all, letter ) { + return letter.toUpperCase(); + }; + +jQuery.fn = jQuery.prototype = { + // The current version of jQuery being used + jquery: version, + + constructor: jQuery, + + // Start with an empty selector + selector: "", + + // The default length of a jQuery object is 0 + length: 0, + + toArray: function() { + return slice.call( this ); + }, + + // Get the Nth element in the matched element set OR + // Get the whole matched element set as a clean array + get: function( num ) { + return num != null ? + + // Return a 'clean' array + ( num < 0 ? this[ num + this.length ] : this[ num ] ) : + + // Return just the object + slice.call( this ); + }, + + // Take an array of elements and push it onto the stack + // (returning the new matched element set) + pushStack: function( elems ) { + + // Build a new jQuery matched element set + var ret = jQuery.merge( this.constructor(), elems ); + + // Add the old object onto the stack (as a reference) + ret.prevObject = this; + ret.context = this.context; + + // Return the newly-formed element set + return ret; + }, + + // Execute a callback for every element in the matched set. + // (You can seed the arguments with an array of args, but this is + // only used internally.) + each: function( callback, args ) { + return jQuery.each( this, callback, args ); + }, + + map: function( callback ) { + return this.pushStack( jQuery.map(this, function( elem, i ) { + return callback.call( elem, i, elem ); + })); + }, + + slice: function() { + return this.pushStack( slice.apply( this, arguments ) ); + }, + + first: function() { + return this.eq( 0 ); + }, + + last: function() { + return this.eq( -1 ); + }, + + eq: function( i ) { + var len = this.length, + j = +i + ( i < 0 ? len : 0 ); + return this.pushStack( j >= 0 && j < len ? [ this[j] ] : [] ); + }, + + end: function() { + return this.prevObject || this.constructor(null); + }, + + // For internal use only. + // Behaves like an Array's method, not like a jQuery method. + push: push, + sort: deletedIds.sort, + splice: deletedIds.splice +}; + +jQuery.extend = jQuery.fn.extend = function() { + var src, copyIsArray, copy, name, options, clone, + target = arguments[0] || {}, + i = 1, + length = arguments.length, + deep = false; + + // Handle a deep copy situation + if ( typeof target === "boolean" ) { + deep = target; + + // skip the boolean and the target + target = arguments[ i ] || {}; + i++; + } + + // Handle case when target is a string or something (possible in deep copy) + if ( typeof target !== "object" && !jQuery.isFunction(target) ) { + target = {}; + } + + // extend jQuery itself if only one argument is passed + if ( i === length ) { + target = this; + i--; + } + + for ( ; i < length; i++ ) { + // Only deal with non-null/undefined values + if ( (options = arguments[ i ]) != null ) { + // Extend the base object + for ( name in options ) { + src = target[ name ]; + copy = options[ name ]; + + // Prevent never-ending loop + if ( target === copy ) { + continue; + } + + // Recurse if we're merging plain objects or arrays + if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) { + if ( copyIsArray ) { + copyIsArray = false; + clone = src && jQuery.isArray(src) ? src : []; + + } else { + clone = src && jQuery.isPlainObject(src) ? src : {}; + } + + // Never move original objects, clone them + target[ name ] = jQuery.extend( deep, clone, copy ); + + // Don't bring in undefined values + } else if ( copy !== undefined ) { + target[ name ] = copy; + } + } + } + } + + // Return the modified object + return target; +}; + +jQuery.extend({ + // Unique for each copy of jQuery on the page + expando: "jQuery" + ( version + Math.random() ).replace( /\D/g, "" ), + + // Assume jQuery is ready without the ready module + isReady: true, + + error: function( msg ) { + throw new Error( msg ); + }, + + noop: function() {}, + + // See test/unit/core.js for details concerning isFunction. + // Since version 1.3, DOM methods and functions like alert + // aren't supported. They return false on IE (#2968). + isFunction: function( obj ) { + return jQuery.type(obj) === "function"; + }, + + isArray: Array.isArray || function( obj ) { + return jQuery.type(obj) === "array"; + }, + + isWindow: function( obj ) { + /* jshint eqeqeq: false */ + return obj != null && obj == obj.window; + }, + + isNumeric: function( obj ) { + // parseFloat NaNs numeric-cast false positives (null|true|false|"") + // ...but misinterprets leading-number strings, particularly hex literals ("0x...") + // subtraction forces infinities to NaN + return obj - parseFloat( obj ) >= 0; + }, + + isEmptyObject: function( obj ) { + var name; + for ( name in obj ) { + return false; + } + return true; + }, + + isPlainObject: function( obj ) { + var key; + + // Must be an Object. + // Because of IE, we also have to check the presence of the constructor property. + // Make sure that DOM nodes and window objects don't pass through, as well + if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) { + return false; + } + + try { + // Not own constructor property must be Object + if ( obj.constructor && + !hasOwn.call(obj, "constructor") && + !hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) { + return false; + } + } catch ( e ) { + // IE8,9 Will throw exceptions on certain host objects #9897 + return false; + } + + // Support: IE<9 + // Handle iteration over inherited properties before own properties. + if ( support.ownLast ) { + for ( key in obj ) { + return hasOwn.call( obj, key ); + } + } + + // Own properties are enumerated firstly, so to speed up, + // if last one is own, then all properties are own. + for ( key in obj ) {} + + return key === undefined || hasOwn.call( obj, key ); + }, + + type: function( obj ) { + if ( obj == null ) { + return obj + ""; + } + return typeof obj === "object" || typeof obj === "function" ? + class2type[ toString.call(obj) ] || "object" : + typeof obj; + }, + + // Evaluates a script in a global context + // Workarounds based on findings by Jim Driscoll + // http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context + globalEval: function( data ) { + if ( data && jQuery.trim( data ) ) { + // We use execScript on Internet Explorer + // We use an anonymous function so that context is window + // rather than jQuery in Firefox + ( window.execScript || function( data ) { + window[ "eval" ].call( window, data ); + } )( data ); + } + }, + + // Convert dashed to camelCase; used by the css and data modules + // Microsoft forgot to hump their vendor prefix (#9572) + camelCase: function( string ) { + return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); + }, + + nodeName: function( elem, name ) { + return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase(); + }, + + // args is for internal usage only + each: function( obj, callback, args ) { + var value, + i = 0, + length = obj.length, + isArray = isArraylike( obj ); + + if ( args ) { + if ( isArray ) { + for ( ; i < length; i++ ) { + value = callback.apply( obj[ i ], args ); + + if ( value === false ) { + break; + } + } + } else { + for ( i in obj ) { + value = callback.apply( obj[ i ], args ); + + if ( value === false ) { + break; + } + } + } + + // A special, fast, case for the most common use of each + } else { + if ( isArray ) { + for ( ; i < length; i++ ) { + value = callback.call( obj[ i ], i, obj[ i ] ); + + if ( value === false ) { + break; + } + } + } else { + for ( i in obj ) { + value = callback.call( obj[ i ], i, obj[ i ] ); + + if ( value === false ) { + break; + } + } + } + } + + return obj; + }, + + // Use native String.trim function wherever possible + trim: trim && !trim.call("\uFEFF\xA0") ? + function( text ) { + return text == null ? + "" : + trim.call( text ); + } : + + // Otherwise use our own trimming functionality + function( text ) { + return text == null ? + "" : + ( text + "" ).replace( rtrim, "" ); + }, + + // results is for internal usage only + makeArray: function( arr, results ) { + var ret = results || []; + + if ( arr != null ) { + if ( isArraylike( Object(arr) ) ) { + jQuery.merge( ret, + typeof arr === "string" ? + [ arr ] : arr + ); + } else { + push.call( ret, arr ); + } + } + + return ret; + }, + + inArray: function( elem, arr, i ) { + var len; + + if ( arr ) { + if ( indexOf ) { + return indexOf.call( arr, elem, i ); + } + + len = arr.length; + i = i ? i < 0 ? Math.max( 0, len + i ) : i : 0; + + for ( ; i < len; i++ ) { + // Skip accessing in sparse arrays + if ( i in arr && arr[ i ] === elem ) { + return i; + } + } + } + + return -1; + }, + + merge: function( first, second ) { + var len = +second.length, + j = 0, + i = first.length; + + while ( j < len ) { + first[ i++ ] = second[ j++ ]; + } + + // Support: IE<9 + // Workaround casting of .length to NaN on otherwise arraylike objects (e.g., NodeLists) + if ( len !== len ) { + while ( second[j] !== undefined ) { + first[ i++ ] = second[ j++ ]; + } + } + + first.length = i; + + return first; + }, + + grep: function( elems, callback, invert ) { + var callbackInverse, + matches = [], + i = 0, + length = elems.length, + callbackExpect = !invert; + + // Go through the array, only saving the items + // that pass the validator function + for ( ; i < length; i++ ) { + callbackInverse = !callback( elems[ i ], i ); + if ( callbackInverse !== callbackExpect ) { + matches.push( elems[ i ] ); + } + } + + return matches; + }, + + // arg is for internal usage only + map: function( elems, callback, arg ) { + var value, + i = 0, + length = elems.length, + isArray = isArraylike( elems ), + ret = []; + + // Go through the array, translating each of the items to their new values + if ( isArray ) { + for ( ; i < length; i++ ) { + value = callback( elems[ i ], i, arg ); + + if ( value != null ) { + ret.push( value ); + } + } + + // Go through every key on the object, + } else { + for ( i in elems ) { + value = callback( elems[ i ], i, arg ); + + if ( value != null ) { + ret.push( value ); + } + } + } + + // Flatten any nested arrays + return concat.apply( [], ret ); + }, + + // A global GUID counter for objects + guid: 1, + + // Bind a function to a context, optionally partially applying any + // arguments. + proxy: function( fn, context ) { + var args, proxy, tmp; + + if ( typeof context === "string" ) { + tmp = fn[ context ]; + context = fn; + fn = tmp; + } + + // Quick check to determine if target is callable, in the spec + // this throws a TypeError, but we will just return undefined. + if ( !jQuery.isFunction( fn ) ) { + return undefined; + } + + // Simulated bind + args = slice.call( arguments, 2 ); + proxy = function() { + return fn.apply( context || this, args.concat( slice.call( arguments ) ) ); + }; + + // Set the guid of unique handler to the same of original handler, so it can be removed + proxy.guid = fn.guid = fn.guid || jQuery.guid++; + + return proxy; + }, + + now: function() { + return +( new Date() ); + }, + + // jQuery.support is not used in Core but other projects attach their + // properties to it so it needs to exist. + support: support +}); + +// Populate the class2type map +jQuery.each("Boolean Number String Function Array Date RegExp Object Error".split(" "), function(i, name) { + class2type[ "[object " + name + "]" ] = name.toLowerCase(); +}); + +function isArraylike( obj ) { + var length = obj.length, + type = jQuery.type( obj ); + + if ( type === "function" || jQuery.isWindow( obj ) ) { + return false; + } + + if ( obj.nodeType === 1 && length ) { + return true; + } + + return type === "array" || length === 0 || + typeof length === "number" && length > 0 && ( length - 1 ) in obj; +} +var Sizzle = +/*! + * Sizzle CSS Selector Engine v1.10.16 + * http://sizzlejs.com/ + * + * Copyright 2013 jQuery Foundation, Inc. and other contributors + * Released under the MIT license + * http://jquery.org/license + * + * Date: 2014-01-13 + */ +(function( window ) { + +var i, + support, + Expr, + getText, + isXML, + compile, + outermostContext, + sortInput, + hasDuplicate, + + // Local document vars + setDocument, + document, + docElem, + documentIsHTML, + rbuggyQSA, + rbuggyMatches, + matches, + contains, + + // Instance-specific data + expando = "sizzle" + -(new Date()), + preferredDoc = window.document, + dirruns = 0, + done = 0, + classCache = createCache(), + tokenCache = createCache(), + compilerCache = createCache(), + sortOrder = function( a, b ) { + if ( a === b ) { + hasDuplicate = true; + } + return 0; + }, + + // General-purpose constants + strundefined = typeof undefined, + MAX_NEGATIVE = 1 << 31, + + // Instance methods + hasOwn = ({}).hasOwnProperty, + arr = [], + pop = arr.pop, + push_native = arr.push, + push = arr.push, + slice = arr.slice, + // Use a stripped-down indexOf if we can't use a native one + indexOf = arr.indexOf || function( elem ) { + var i = 0, + len = this.length; + for ( ; i < len; i++ ) { + if ( this[i] === elem ) { + return i; + } + } + return -1; + }, + + booleans = "checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped", + + // Regular expressions + + // Whitespace characters http://www.w3.org/TR/css3-selectors/#whitespace + whitespace = "[\\x20\\t\\r\\n\\f]", + // http://www.w3.org/TR/css3-syntax/#characters + characterEncoding = "(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+", + + // Loosely modeled on CSS identifier characters + // An unquoted value should be a CSS identifier http://www.w3.org/TR/css3-selectors/#attribute-selectors + // Proper syntax: http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier + identifier = characterEncoding.replace( "w", "w#" ), + + // Acceptable operators http://www.w3.org/TR/selectors/#attribute-selectors + attributes = "\\[" + whitespace + "*(" + characterEncoding + ")" + whitespace + + "*(?:([*^$|!~]?=)" + whitespace + "*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|(" + identifier + ")|)|)" + whitespace + "*\\]", + + // Prefer arguments quoted, + // then not containing pseudos/brackets, + // then attribute selectors/non-parenthetical expressions, + // then anything else + // These preferences are here to reduce the number of selectors + // needing tokenize in the PSEUDO preFilter + pseudos = ":(" + characterEncoding + ")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|" + attributes.replace( 3, 8 ) + ")*)|.*)\\)|)", + + // Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter + rtrim = new RegExp( "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$", "g" ), + + rcomma = new RegExp( "^" + whitespace + "*," + whitespace + "*" ), + rcombinators = new RegExp( "^" + whitespace + "*([>+~]|" + whitespace + ")" + whitespace + "*" ), + + rattributeQuotes = new RegExp( "=" + whitespace + "*([^\\]'\"]*?)" + whitespace + "*\\]", "g" ), + + rpseudo = new RegExp( pseudos ), + ridentifier = new RegExp( "^" + identifier + "$" ), + + matchExpr = { + "ID": new RegExp( "^#(" + characterEncoding + ")" ), + "CLASS": new RegExp( "^\\.(" + characterEncoding + ")" ), + "TAG": new RegExp( "^(" + characterEncoding.replace( "w", "w*" ) + ")" ), + "ATTR": new RegExp( "^" + attributes ), + "PSEUDO": new RegExp( "^" + pseudos ), + "CHILD": new RegExp( "^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\(" + whitespace + + "*(even|odd|(([+-]|)(\\d*)n|)" + whitespace + "*(?:([+-]|)" + whitespace + + "*(\\d+)|))" + whitespace + "*\\)|)", "i" ), + "bool": new RegExp( "^(?:" + booleans + ")$", "i" ), + // For use in libraries implementing .is() + // We use this for POS matching in `select` + "needsContext": new RegExp( "^" + whitespace + "*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\(" + + whitespace + "*((?:-\\d)?\\d*)" + whitespace + "*\\)|)(?=[^-]|$)", "i" ) + }, + + rinputs = /^(?:input|select|textarea|button)$/i, + rheader = /^h\d$/i, + + rnative = /^[^{]+\{\s*\[native \w/, + + // Easily-parseable/retrievable ID or TAG or CLASS selectors + rquickExpr = /^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/, + + rsibling = /[+~]/, + rescape = /'|\\/g, + + // CSS escapes http://www.w3.org/TR/CSS21/syndata.html#escaped-characters + runescape = new RegExp( "\\\\([\\da-f]{1,6}" + whitespace + "?|(" + whitespace + ")|.)", "ig" ), + funescape = function( _, escaped, escapedWhitespace ) { + var high = "0x" + escaped - 0x10000; + // NaN means non-codepoint + // Support: Firefox + // Workaround erroneous numeric interpretation of +"0x" + return high !== high || escapedWhitespace ? + escaped : + high < 0 ? + // BMP codepoint + String.fromCharCode( high + 0x10000 ) : + // Supplemental Plane codepoint (surrogate pair) + String.fromCharCode( high >> 10 | 0xD800, high & 0x3FF | 0xDC00 ); + }; + +// Optimize for push.apply( _, NodeList ) +try { + push.apply( + (arr = slice.call( preferredDoc.childNodes )), + preferredDoc.childNodes + ); + // Support: Android<4.0 + // Detect silently failing push.apply + arr[ preferredDoc.childNodes.length ].nodeType; +} catch ( e ) { + push = { apply: arr.length ? + + // Leverage slice if possible + function( target, els ) { + push_native.apply( target, slice.call(els) ); + } : + + // Support: IE<9 + // Otherwise append directly + function( target, els ) { + var j = target.length, + i = 0; + // Can't trust NodeList.length + while ( (target[j++] = els[i++]) ) {} + target.length = j - 1; + } + }; +} + +function Sizzle( selector, context, results, seed ) { + var match, elem, m, nodeType, + // QSA vars + i, groups, old, nid, newContext, newSelector; + + if ( ( context ? context.ownerDocument || context : preferredDoc ) !== document ) { + setDocument( context ); + } + + context = context || document; + results = results || []; + + if ( !selector || typeof selector !== "string" ) { + return results; + } + + if ( (nodeType = context.nodeType) !== 1 && nodeType !== 9 ) { + return []; + } + + if ( documentIsHTML && !seed ) { + + // Shortcuts + if ( (match = rquickExpr.exec( selector )) ) { + // Speed-up: Sizzle("#ID") + if ( (m = match[1]) ) { + if ( nodeType === 9 ) { + elem = context.getElementById( m ); + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document (jQuery #6963) + if ( elem && elem.parentNode ) { + // Handle the case where IE, Opera, and Webkit return items + // by name instead of ID + if ( elem.id === m ) { + results.push( elem ); + return results; + } + } else { + return results; + } + } else { + // Context is not a document + if ( context.ownerDocument && (elem = context.ownerDocument.getElementById( m )) && + contains( context, elem ) && elem.id === m ) { + results.push( elem ); + return results; + } + } + + // Speed-up: Sizzle("TAG") + } else if ( match[2] ) { + push.apply( results, context.getElementsByTagName( selector ) ); + return results; + + // Speed-up: Sizzle(".CLASS") + } else if ( (m = match[3]) && support.getElementsByClassName && context.getElementsByClassName ) { + push.apply( results, context.getElementsByClassName( m ) ); + return results; + } + } + + // QSA path + if ( support.qsa && (!rbuggyQSA || !rbuggyQSA.test( selector )) ) { + nid = old = expando; + newContext = context; + newSelector = nodeType === 9 && selector; + + // qSA works strangely on Element-rooted queries + // We can work around this by specifying an extra ID on the root + // and working up from there (Thanks to Andrew Dupont for the technique) + // IE 8 doesn't work on object elements + if ( nodeType === 1 && context.nodeName.toLowerCase() !== "object" ) { + groups = tokenize( selector ); + + if ( (old = context.getAttribute("id")) ) { + nid = old.replace( rescape, "\\$&" ); + } else { + context.setAttribute( "id", nid ); + } + nid = "[id='" + nid + "'] "; + + i = groups.length; + while ( i-- ) { + groups[i] = nid + toSelector( groups[i] ); + } + newContext = rsibling.test( selector ) && testContext( context.parentNode ) || context; + newSelector = groups.join(","); + } + + if ( newSelector ) { + try { + push.apply( results, + newContext.querySelectorAll( newSelector ) + ); + return results; + } catch(qsaError) { + } finally { + if ( !old ) { + context.removeAttribute("id"); + } + } + } + } + } + + // All others + return select( selector.replace( rtrim, "$1" ), context, results, seed ); +} + +/** + * Create key-value caches of limited size + * @returns {Function(string, Object)} Returns the Object data after storing it on itself with + * property name the (space-suffixed) string and (if the cache is larger than Expr.cacheLength) + * deleting the oldest entry + */ +function createCache() { + var keys = []; + + function cache( key, value ) { + // Use (key + " ") to avoid collision with native prototype properties (see Issue #157) + if ( keys.push( key + " " ) > Expr.cacheLength ) { + // Only keep the most recent entries + delete cache[ keys.shift() ]; + } + return (cache[ key + " " ] = value); + } + return cache; +} + +/** + * Mark a function for special use by Sizzle + * @param {Function} fn The function to mark + */ +function markFunction( fn ) { + fn[ expando ] = true; + return fn; +} + +/** + * Support testing using an element + * @param {Function} fn Passed the created div and expects a boolean result + */ +function assert( fn ) { + var div = document.createElement("div"); + + try { + return !!fn( div ); + } catch (e) { + return false; + } finally { + // Remove from its parent by default + if ( div.parentNode ) { + div.parentNode.removeChild( div ); + } + // release memory in IE + div = null; + } +} + +/** + * Adds the same handler for all of the specified attrs + * @param {String} attrs Pipe-separated list of attributes + * @param {Function} handler The method that will be applied + */ +function addHandle( attrs, handler ) { + var arr = attrs.split("|"), + i = attrs.length; + + while ( i-- ) { + Expr.attrHandle[ arr[i] ] = handler; + } +} + +/** + * Checks document order of two siblings + * @param {Element} a + * @param {Element} b + * @returns {Number} Returns less than 0 if a precedes b, greater than 0 if a follows b + */ +function siblingCheck( a, b ) { + var cur = b && a, + diff = cur && a.nodeType === 1 && b.nodeType === 1 && + ( ~b.sourceIndex || MAX_NEGATIVE ) - + ( ~a.sourceIndex || MAX_NEGATIVE ); + + // Use IE sourceIndex if available on both nodes + if ( diff ) { + return diff; + } + + // Check if b follows a + if ( cur ) { + while ( (cur = cur.nextSibling) ) { + if ( cur === b ) { + return -1; + } + } + } + + return a ? 1 : -1; +} + +/** + * Returns a function to use in pseudos for input types + * @param {String} type + */ +function createInputPseudo( type ) { + return function( elem ) { + var name = elem.nodeName.toLowerCase(); + return name === "input" && elem.type === type; + }; +} + +/** + * Returns a function to use in pseudos for buttons + * @param {String} type + */ +function createButtonPseudo( type ) { + return function( elem ) { + var name = elem.nodeName.toLowerCase(); + return (name === "input" || name === "button") && elem.type === type; + }; +} + +/** + * Returns a function to use in pseudos for positionals + * @param {Function} fn + */ +function createPositionalPseudo( fn ) { + return markFunction(function( argument ) { + argument = +argument; + return markFunction(function( seed, matches ) { + var j, + matchIndexes = fn( [], seed.length, argument ), + i = matchIndexes.length; + + // Match elements found at the specified indexes + while ( i-- ) { + if ( seed[ (j = matchIndexes[i]) ] ) { + seed[j] = !(matches[j] = seed[j]); + } + } + }); + }); +} + +/** + * Checks a node for validity as a Sizzle context + * @param {Element|Object=} context + * @returns {Element|Object|Boolean} The input node if acceptable, otherwise a falsy value + */ +function testContext( context ) { + return context && typeof context.getElementsByTagName !== strundefined && context; +} + +// Expose support vars for convenience +support = Sizzle.support = {}; + +/** + * Detects XML nodes + * @param {Element|Object} elem An element or a document + * @returns {Boolean} True iff elem is a non-HTML XML node + */ +isXML = Sizzle.isXML = function( elem ) { + // documentElement is verified for cases where it doesn't yet exist + // (such as loading iframes in IE - #4833) + var documentElement = elem && (elem.ownerDocument || elem).documentElement; + return documentElement ? documentElement.nodeName !== "HTML" : false; +}; + +/** + * Sets document-related variables once based on the current document + * @param {Element|Object} [doc] An element or document object to use to set the document + * @returns {Object} Returns the current document + */ +setDocument = Sizzle.setDocument = function( node ) { + var hasCompare, + doc = node ? node.ownerDocument || node : preferredDoc, + parent = doc.defaultView; + + // If no document and documentElement is available, return + if ( doc === document || doc.nodeType !== 9 || !doc.documentElement ) { + return document; + } + + // Set our document + document = doc; + docElem = doc.documentElement; + + // Support tests + documentIsHTML = !isXML( doc ); + + // Support: IE>8 + // If iframe document is assigned to "document" variable and if iframe has been reloaded, + // IE will throw "permission denied" error when accessing "document" variable, see jQuery #13936 + // IE6-8 do not support the defaultView property so parent will be undefined + if ( parent && parent !== parent.top ) { + // IE11 does not have attachEvent, so all must suffer + if ( parent.addEventListener ) { + parent.addEventListener( "unload", function() { + setDocument(); + }, false ); + } else if ( parent.attachEvent ) { + parent.attachEvent( "onunload", function() { + setDocument(); + }); + } + } + + /* Attributes + ---------------------------------------------------------------------- */ + + // Support: IE<8 + // Verify that getAttribute really returns attributes and not properties (excepting IE8 booleans) + support.attributes = assert(function( div ) { + div.className = "i"; + return !div.getAttribute("className"); + }); + + /* getElement(s)By* + ---------------------------------------------------------------------- */ + + // Check if getElementsByTagName("*") returns only elements + support.getElementsByTagName = assert(function( div ) { + div.appendChild( doc.createComment("") ); + return !div.getElementsByTagName("*").length; + }); + + // Check if getElementsByClassName can be trusted + support.getElementsByClassName = rnative.test( doc.getElementsByClassName ) && assert(function( div ) { + div.innerHTML = "
    "; + + // Support: Safari<4 + // Catch class over-caching + div.firstChild.className = "i"; + // Support: Opera<10 + // Catch gEBCN failure to find non-leading classes + return div.getElementsByClassName("i").length === 2; + }); + + // Support: IE<10 + // Check if getElementById returns elements by name + // The broken getElementById methods don't pick up programatically-set names, + // so use a roundabout getElementsByName test + support.getById = assert(function( div ) { + docElem.appendChild( div ).id = expando; + return !doc.getElementsByName || !doc.getElementsByName( expando ).length; + }); + + // ID find and filter + if ( support.getById ) { + Expr.find["ID"] = function( id, context ) { + if ( typeof context.getElementById !== strundefined && documentIsHTML ) { + var m = context.getElementById( id ); + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + return m && m.parentNode ? [m] : []; + } + }; + Expr.filter["ID"] = function( id ) { + var attrId = id.replace( runescape, funescape ); + return function( elem ) { + return elem.getAttribute("id") === attrId; + }; + }; + } else { + // Support: IE6/7 + // getElementById is not reliable as a find shortcut + delete Expr.find["ID"]; + + Expr.filter["ID"] = function( id ) { + var attrId = id.replace( runescape, funescape ); + return function( elem ) { + var node = typeof elem.getAttributeNode !== strundefined && elem.getAttributeNode("id"); + return node && node.value === attrId; + }; + }; + } + + // Tag + Expr.find["TAG"] = support.getElementsByTagName ? + function( tag, context ) { + if ( typeof context.getElementsByTagName !== strundefined ) { + return context.getElementsByTagName( tag ); + } + } : + function( tag, context ) { + var elem, + tmp = [], + i = 0, + results = context.getElementsByTagName( tag ); + + // Filter out possible comments + if ( tag === "*" ) { + while ( (elem = results[i++]) ) { + if ( elem.nodeType === 1 ) { + tmp.push( elem ); + } + } + + return tmp; + } + return results; + }; + + // Class + Expr.find["CLASS"] = support.getElementsByClassName && function( className, context ) { + if ( typeof context.getElementsByClassName !== strundefined && documentIsHTML ) { + return context.getElementsByClassName( className ); + } + }; + + /* QSA/matchesSelector + ---------------------------------------------------------------------- */ + + // QSA and matchesSelector support + + // matchesSelector(:active) reports false when true (IE9/Opera 11.5) + rbuggyMatches = []; + + // qSa(:focus) reports false when true (Chrome 21) + // We allow this because of a bug in IE8/9 that throws an error + // whenever `document.activeElement` is accessed on an iframe + // So, we allow :focus to pass through QSA all the time to avoid the IE error + // See http://bugs.jquery.com/ticket/13378 + rbuggyQSA = []; + + if ( (support.qsa = rnative.test( doc.querySelectorAll )) ) { + // Build QSA regex + // Regex strategy adopted from Diego Perini + assert(function( div ) { + // Select is set to empty string on purpose + // This is to test IE's treatment of not explicitly + // setting a boolean content attribute, + // since its presence should be enough + // http://bugs.jquery.com/ticket/12359 + div.innerHTML = ""; + + // Support: IE8, Opera 10-12 + // Nothing should be selected when empty strings follow ^= or $= or *= + if ( div.querySelectorAll("[t^='']").length ) { + rbuggyQSA.push( "[*^$]=" + whitespace + "*(?:''|\"\")" ); + } + + // Support: IE8 + // Boolean attributes and "value" are not treated correctly + if ( !div.querySelectorAll("[selected]").length ) { + rbuggyQSA.push( "\\[" + whitespace + "*(?:value|" + booleans + ")" ); + } + + // Webkit/Opera - :checked should return selected option elements + // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked + // IE8 throws error here and will not see later tests + if ( !div.querySelectorAll(":checked").length ) { + rbuggyQSA.push(":checked"); + } + }); + + assert(function( div ) { + // Support: Windows 8 Native Apps + // The type and name attributes are restricted during .innerHTML assignment + var input = doc.createElement("input"); + input.setAttribute( "type", "hidden" ); + div.appendChild( input ).setAttribute( "name", "D" ); + + // Support: IE8 + // Enforce case-sensitivity of name attribute + if ( div.querySelectorAll("[name=d]").length ) { + rbuggyQSA.push( "name" + whitespace + "*[*^$|!~]?=" ); + } + + // FF 3.5 - :enabled/:disabled and hidden elements (hidden elements are still enabled) + // IE8 throws error here and will not see later tests + if ( !div.querySelectorAll(":enabled").length ) { + rbuggyQSA.push( ":enabled", ":disabled" ); + } + + // Opera 10-11 does not throw on post-comma invalid pseudos + div.querySelectorAll("*,:x"); + rbuggyQSA.push(",.*:"); + }); + } + + if ( (support.matchesSelector = rnative.test( (matches = docElem.webkitMatchesSelector || + docElem.mozMatchesSelector || + docElem.oMatchesSelector || + docElem.msMatchesSelector) )) ) { + + assert(function( div ) { + // Check to see if it's possible to do matchesSelector + // on a disconnected node (IE 9) + support.disconnectedMatch = matches.call( div, "div" ); + + // This should fail with an exception + // Gecko does not error, returns false instead + matches.call( div, "[s!='']:x" ); + rbuggyMatches.push( "!=", pseudos ); + }); + } + + rbuggyQSA = rbuggyQSA.length && new RegExp( rbuggyQSA.join("|") ); + rbuggyMatches = rbuggyMatches.length && new RegExp( rbuggyMatches.join("|") ); + + /* Contains + ---------------------------------------------------------------------- */ + hasCompare = rnative.test( docElem.compareDocumentPosition ); + + // Element contains another + // Purposefully does not implement inclusive descendent + // As in, an element does not contain itself + contains = hasCompare || rnative.test( docElem.contains ) ? + function( a, b ) { + var adown = a.nodeType === 9 ? a.documentElement : a, + bup = b && b.parentNode; + return a === bup || !!( bup && bup.nodeType === 1 && ( + adown.contains ? + adown.contains( bup ) : + a.compareDocumentPosition && a.compareDocumentPosition( bup ) & 16 + )); + } : + function( a, b ) { + if ( b ) { + while ( (b = b.parentNode) ) { + if ( b === a ) { + return true; + } + } + } + return false; + }; + + /* Sorting + ---------------------------------------------------------------------- */ + + // Document order sorting + sortOrder = hasCompare ? + function( a, b ) { + + // Flag for duplicate removal + if ( a === b ) { + hasDuplicate = true; + return 0; + } + + // Sort on method existence if only one input has compareDocumentPosition + var compare = !a.compareDocumentPosition - !b.compareDocumentPosition; + if ( compare ) { + return compare; + } + + // Calculate position if both inputs belong to the same document + compare = ( a.ownerDocument || a ) === ( b.ownerDocument || b ) ? + a.compareDocumentPosition( b ) : + + // Otherwise we know they are disconnected + 1; + + // Disconnected nodes + if ( compare & 1 || + (!support.sortDetached && b.compareDocumentPosition( a ) === compare) ) { + + // Choose the first element that is related to our preferred document + if ( a === doc || a.ownerDocument === preferredDoc && contains(preferredDoc, a) ) { + return -1; + } + if ( b === doc || b.ownerDocument === preferredDoc && contains(preferredDoc, b) ) { + return 1; + } + + // Maintain original order + return sortInput ? + ( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) : + 0; + } + + return compare & 4 ? -1 : 1; + } : + function( a, b ) { + // Exit early if the nodes are identical + if ( a === b ) { + hasDuplicate = true; + return 0; + } + + var cur, + i = 0, + aup = a.parentNode, + bup = b.parentNode, + ap = [ a ], + bp = [ b ]; + + // Parentless nodes are either documents or disconnected + if ( !aup || !bup ) { + return a === doc ? -1 : + b === doc ? 1 : + aup ? -1 : + bup ? 1 : + sortInput ? + ( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) : + 0; + + // If the nodes are siblings, we can do a quick check + } else if ( aup === bup ) { + return siblingCheck( a, b ); + } + + // Otherwise we need full lists of their ancestors for comparison + cur = a; + while ( (cur = cur.parentNode) ) { + ap.unshift( cur ); + } + cur = b; + while ( (cur = cur.parentNode) ) { + bp.unshift( cur ); + } + + // Walk down the tree looking for a discrepancy + while ( ap[i] === bp[i] ) { + i++; + } + + return i ? + // Do a sibling check if the nodes have a common ancestor + siblingCheck( ap[i], bp[i] ) : + + // Otherwise nodes in our document sort first + ap[i] === preferredDoc ? -1 : + bp[i] === preferredDoc ? 1 : + 0; + }; + + return doc; +}; + +Sizzle.matches = function( expr, elements ) { + return Sizzle( expr, null, null, elements ); +}; + +Sizzle.matchesSelector = function( elem, expr ) { + // Set document vars if needed + if ( ( elem.ownerDocument || elem ) !== document ) { + setDocument( elem ); + } + + // Make sure that attribute selectors are quoted + expr = expr.replace( rattributeQuotes, "='$1']" ); + + if ( support.matchesSelector && documentIsHTML && + ( !rbuggyMatches || !rbuggyMatches.test( expr ) ) && + ( !rbuggyQSA || !rbuggyQSA.test( expr ) ) ) { + + try { + var ret = matches.call( elem, expr ); + + // IE 9's matchesSelector returns false on disconnected nodes + if ( ret || support.disconnectedMatch || + // As well, disconnected nodes are said to be in a document + // fragment in IE 9 + elem.document && elem.document.nodeType !== 11 ) { + return ret; + } + } catch(e) {} + } + + return Sizzle( expr, document, null, [elem] ).length > 0; +}; + +Sizzle.contains = function( context, elem ) { + // Set document vars if needed + if ( ( context.ownerDocument || context ) !== document ) { + setDocument( context ); + } + return contains( context, elem ); +}; + +Sizzle.attr = function( elem, name ) { + // Set document vars if needed + if ( ( elem.ownerDocument || elem ) !== document ) { + setDocument( elem ); + } + + var fn = Expr.attrHandle[ name.toLowerCase() ], + // Don't get fooled by Object.prototype properties (jQuery #13807) + val = fn && hasOwn.call( Expr.attrHandle, name.toLowerCase() ) ? + fn( elem, name, !documentIsHTML ) : + undefined; + + return val !== undefined ? + val : + support.attributes || !documentIsHTML ? + elem.getAttribute( name ) : + (val = elem.getAttributeNode(name)) && val.specified ? + val.value : + null; +}; + +Sizzle.error = function( msg ) { + throw new Error( "Syntax error, unrecognized expression: " + msg ); +}; + +/** + * Document sorting and removing duplicates + * @param {ArrayLike} results + */ +Sizzle.uniqueSort = function( results ) { + var elem, + duplicates = [], + j = 0, + i = 0; + + // Unless we *know* we can detect duplicates, assume their presence + hasDuplicate = !support.detectDuplicates; + sortInput = !support.sortStable && results.slice( 0 ); + results.sort( sortOrder ); + + if ( hasDuplicate ) { + while ( (elem = results[i++]) ) { + if ( elem === results[ i ] ) { + j = duplicates.push( i ); + } + } + while ( j-- ) { + results.splice( duplicates[ j ], 1 ); + } + } + + // Clear input after sorting to release objects + // See https://github.com/jquery/sizzle/pull/225 + sortInput = null; + + return results; +}; + +/** + * Utility function for retrieving the text value of an array of DOM nodes + * @param {Array|Element} elem + */ +getText = Sizzle.getText = function( elem ) { + var node, + ret = "", + i = 0, + nodeType = elem.nodeType; + + if ( !nodeType ) { + // If no nodeType, this is expected to be an array + while ( (node = elem[i++]) ) { + // Do not traverse comment nodes + ret += getText( node ); + } + } else if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) { + // Use textContent for elements + // innerText usage removed for consistency of new lines (jQuery #11153) + if ( typeof elem.textContent === "string" ) { + return elem.textContent; + } else { + // Traverse its children + for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { + ret += getText( elem ); + } + } + } else if ( nodeType === 3 || nodeType === 4 ) { + return elem.nodeValue; + } + // Do not include comment or processing instruction nodes + + return ret; +}; + +Expr = Sizzle.selectors = { + + // Can be adjusted by the user + cacheLength: 50, + + createPseudo: markFunction, + + match: matchExpr, + + attrHandle: {}, + + find: {}, + + relative: { + ">": { dir: "parentNode", first: true }, + " ": { dir: "parentNode" }, + "+": { dir: "previousSibling", first: true }, + "~": { dir: "previousSibling" } + }, + + preFilter: { + "ATTR": function( match ) { + match[1] = match[1].replace( runescape, funescape ); + + // Move the given value to match[3] whether quoted or unquoted + match[3] = ( match[4] || match[5] || "" ).replace( runescape, funescape ); + + if ( match[2] === "~=" ) { + match[3] = " " + match[3] + " "; + } + + return match.slice( 0, 4 ); + }, + + "CHILD": function( match ) { + /* matches from matchExpr["CHILD"] + 1 type (only|nth|...) + 2 what (child|of-type) + 3 argument (even|odd|\d*|\d*n([+-]\d+)?|...) + 4 xn-component of xn+y argument ([+-]?\d*n|) + 5 sign of xn-component + 6 x of xn-component + 7 sign of y-component + 8 y of y-component + */ + match[1] = match[1].toLowerCase(); + + if ( match[1].slice( 0, 3 ) === "nth" ) { + // nth-* requires argument + if ( !match[3] ) { + Sizzle.error( match[0] ); + } + + // numeric x and y parameters for Expr.filter.CHILD + // remember that false/true cast respectively to 0/1 + match[4] = +( match[4] ? match[5] + (match[6] || 1) : 2 * ( match[3] === "even" || match[3] === "odd" ) ); + match[5] = +( ( match[7] + match[8] ) || match[3] === "odd" ); + + // other types prohibit arguments + } else if ( match[3] ) { + Sizzle.error( match[0] ); + } + + return match; + }, + + "PSEUDO": function( match ) { + var excess, + unquoted = !match[5] && match[2]; + + if ( matchExpr["CHILD"].test( match[0] ) ) { + return null; + } + + // Accept quoted arguments as-is + if ( match[3] && match[4] !== undefined ) { + match[2] = match[4]; + + // Strip excess characters from unquoted arguments + } else if ( unquoted && rpseudo.test( unquoted ) && + // Get excess from tokenize (recursively) + (excess = tokenize( unquoted, true )) && + // advance to the next closing parenthesis + (excess = unquoted.indexOf( ")", unquoted.length - excess ) - unquoted.length) ) { + + // excess is a negative index + match[0] = match[0].slice( 0, excess ); + match[2] = unquoted.slice( 0, excess ); + } + + // Return only captures needed by the pseudo filter method (type and argument) + return match.slice( 0, 3 ); + } + }, + + filter: { + + "TAG": function( nodeNameSelector ) { + var nodeName = nodeNameSelector.replace( runescape, funescape ).toLowerCase(); + return nodeNameSelector === "*" ? + function() { return true; } : + function( elem ) { + return elem.nodeName && elem.nodeName.toLowerCase() === nodeName; + }; + }, + + "CLASS": function( className ) { + var pattern = classCache[ className + " " ]; + + return pattern || + (pattern = new RegExp( "(^|" + whitespace + ")" + className + "(" + whitespace + "|$)" )) && + classCache( className, function( elem ) { + return pattern.test( typeof elem.className === "string" && elem.className || typeof elem.getAttribute !== strundefined && elem.getAttribute("class") || "" ); + }); + }, + + "ATTR": function( name, operator, check ) { + return function( elem ) { + var result = Sizzle.attr( elem, name ); + + if ( result == null ) { + return operator === "!="; + } + if ( !operator ) { + return true; + } + + result += ""; + + return operator === "=" ? result === check : + operator === "!=" ? result !== check : + operator === "^=" ? check && result.indexOf( check ) === 0 : + operator === "*=" ? check && result.indexOf( check ) > -1 : + operator === "$=" ? check && result.slice( -check.length ) === check : + operator === "~=" ? ( " " + result + " " ).indexOf( check ) > -1 : + operator === "|=" ? result === check || result.slice( 0, check.length + 1 ) === check + "-" : + false; + }; + }, + + "CHILD": function( type, what, argument, first, last ) { + var simple = type.slice( 0, 3 ) !== "nth", + forward = type.slice( -4 ) !== "last", + ofType = what === "of-type"; + + return first === 1 && last === 0 ? + + // Shortcut for :nth-*(n) + function( elem ) { + return !!elem.parentNode; + } : + + function( elem, context, xml ) { + var cache, outerCache, node, diff, nodeIndex, start, + dir = simple !== forward ? "nextSibling" : "previousSibling", + parent = elem.parentNode, + name = ofType && elem.nodeName.toLowerCase(), + useCache = !xml && !ofType; + + if ( parent ) { + + // :(first|last|only)-(child|of-type) + if ( simple ) { + while ( dir ) { + node = elem; + while ( (node = node[ dir ]) ) { + if ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) { + return false; + } + } + // Reverse direction for :only-* (if we haven't yet done so) + start = dir = type === "only" && !start && "nextSibling"; + } + return true; + } + + start = [ forward ? parent.firstChild : parent.lastChild ]; + + // non-xml :nth-child(...) stores cache data on `parent` + if ( forward && useCache ) { + // Seek `elem` from a previously-cached index + outerCache = parent[ expando ] || (parent[ expando ] = {}); + cache = outerCache[ type ] || []; + nodeIndex = cache[0] === dirruns && cache[1]; + diff = cache[0] === dirruns && cache[2]; + node = nodeIndex && parent.childNodes[ nodeIndex ]; + + while ( (node = ++nodeIndex && node && node[ dir ] || + + // Fallback to seeking `elem` from the start + (diff = nodeIndex = 0) || start.pop()) ) { + + // When found, cache indexes on `parent` and break + if ( node.nodeType === 1 && ++diff && node === elem ) { + outerCache[ type ] = [ dirruns, nodeIndex, diff ]; + break; + } + } + + // Use previously-cached element index if available + } else if ( useCache && (cache = (elem[ expando ] || (elem[ expando ] = {}))[ type ]) && cache[0] === dirruns ) { + diff = cache[1]; + + // xml :nth-child(...) or :nth-last-child(...) or :nth(-last)?-of-type(...) + } else { + // Use the same loop as above to seek `elem` from the start + while ( (node = ++nodeIndex && node && node[ dir ] || + (diff = nodeIndex = 0) || start.pop()) ) { + + if ( ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) && ++diff ) { + // Cache the index of each encountered element + if ( useCache ) { + (node[ expando ] || (node[ expando ] = {}))[ type ] = [ dirruns, diff ]; + } + + if ( node === elem ) { + break; + } + } + } + } + + // Incorporate the offset, then check against cycle size + diff -= last; + return diff === first || ( diff % first === 0 && diff / first >= 0 ); + } + }; + }, + + "PSEUDO": function( pseudo, argument ) { + // pseudo-class names are case-insensitive + // http://www.w3.org/TR/selectors/#pseudo-classes + // Prioritize by case sensitivity in case custom pseudos are added with uppercase letters + // Remember that setFilters inherits from pseudos + var args, + fn = Expr.pseudos[ pseudo ] || Expr.setFilters[ pseudo.toLowerCase() ] || + Sizzle.error( "unsupported pseudo: " + pseudo ); + + // The user may use createPseudo to indicate that + // arguments are needed to create the filter function + // just as Sizzle does + if ( fn[ expando ] ) { + return fn( argument ); + } + + // But maintain support for old signatures + if ( fn.length > 1 ) { + args = [ pseudo, pseudo, "", argument ]; + return Expr.setFilters.hasOwnProperty( pseudo.toLowerCase() ) ? + markFunction(function( seed, matches ) { + var idx, + matched = fn( seed, argument ), + i = matched.length; + while ( i-- ) { + idx = indexOf.call( seed, matched[i] ); + seed[ idx ] = !( matches[ idx ] = matched[i] ); + } + }) : + function( elem ) { + return fn( elem, 0, args ); + }; + } + + return fn; + } + }, + + pseudos: { + // Potentially complex pseudos + "not": markFunction(function( selector ) { + // Trim the selector passed to compile + // to avoid treating leading and trailing + // spaces as combinators + var input = [], + results = [], + matcher = compile( selector.replace( rtrim, "$1" ) ); + + return matcher[ expando ] ? + markFunction(function( seed, matches, context, xml ) { + var elem, + unmatched = matcher( seed, null, xml, [] ), + i = seed.length; + + // Match elements unmatched by `matcher` + while ( i-- ) { + if ( (elem = unmatched[i]) ) { + seed[i] = !(matches[i] = elem); + } + } + }) : + function( elem, context, xml ) { + input[0] = elem; + matcher( input, null, xml, results ); + return !results.pop(); + }; + }), + + "has": markFunction(function( selector ) { + return function( elem ) { + return Sizzle( selector, elem ).length > 0; + }; + }), + + "contains": markFunction(function( text ) { + return function( elem ) { + return ( elem.textContent || elem.innerText || getText( elem ) ).indexOf( text ) > -1; + }; + }), + + // "Whether an element is represented by a :lang() selector + // is based solely on the element's language value + // being equal to the identifier C, + // or beginning with the identifier C immediately followed by "-". + // The matching of C against the element's language value is performed case-insensitively. + // The identifier C does not have to be a valid language name." + // http://www.w3.org/TR/selectors/#lang-pseudo + "lang": markFunction( function( lang ) { + // lang value must be a valid identifier + if ( !ridentifier.test(lang || "") ) { + Sizzle.error( "unsupported lang: " + lang ); + } + lang = lang.replace( runescape, funescape ).toLowerCase(); + return function( elem ) { + var elemLang; + do { + if ( (elemLang = documentIsHTML ? + elem.lang : + elem.getAttribute("xml:lang") || elem.getAttribute("lang")) ) { + + elemLang = elemLang.toLowerCase(); + return elemLang === lang || elemLang.indexOf( lang + "-" ) === 0; + } + } while ( (elem = elem.parentNode) && elem.nodeType === 1 ); + return false; + }; + }), + + // Miscellaneous + "target": function( elem ) { + var hash = window.location && window.location.hash; + return hash && hash.slice( 1 ) === elem.id; + }, + + "root": function( elem ) { + return elem === docElem; + }, + + "focus": function( elem ) { + return elem === document.activeElement && (!document.hasFocus || document.hasFocus()) && !!(elem.type || elem.href || ~elem.tabIndex); + }, + + // Boolean properties + "enabled": function( elem ) { + return elem.disabled === false; + }, + + "disabled": function( elem ) { + return elem.disabled === true; + }, + + "checked": function( elem ) { + // In CSS3, :checked should return both checked and selected elements + // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked + var nodeName = elem.nodeName.toLowerCase(); + return (nodeName === "input" && !!elem.checked) || (nodeName === "option" && !!elem.selected); + }, + + "selected": function( elem ) { + // Accessing this property makes selected-by-default + // options in Safari work properly + if ( elem.parentNode ) { + elem.parentNode.selectedIndex; + } + + return elem.selected === true; + }, + + // Contents + "empty": function( elem ) { + // http://www.w3.org/TR/selectors/#empty-pseudo + // :empty is negated by element (1) or content nodes (text: 3; cdata: 4; entity ref: 5), + // but not by others (comment: 8; processing instruction: 7; etc.) + // nodeType < 6 works because attributes (2) do not appear as children + for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { + if ( elem.nodeType < 6 ) { + return false; + } + } + return true; + }, + + "parent": function( elem ) { + return !Expr.pseudos["empty"]( elem ); + }, + + // Element/input types + "header": function( elem ) { + return rheader.test( elem.nodeName ); + }, + + "input": function( elem ) { + return rinputs.test( elem.nodeName ); + }, + + "button": function( elem ) { + var name = elem.nodeName.toLowerCase(); + return name === "input" && elem.type === "button" || name === "button"; + }, + + "text": function( elem ) { + var attr; + return elem.nodeName.toLowerCase() === "input" && + elem.type === "text" && + + // Support: IE<8 + // New HTML5 attribute values (e.g., "search") appear with elem.type === "text" + ( (attr = elem.getAttribute("type")) == null || attr.toLowerCase() === "text" ); + }, + + // Position-in-collection + "first": createPositionalPseudo(function() { + return [ 0 ]; + }), + + "last": createPositionalPseudo(function( matchIndexes, length ) { + return [ length - 1 ]; + }), + + "eq": createPositionalPseudo(function( matchIndexes, length, argument ) { + return [ argument < 0 ? argument + length : argument ]; + }), + + "even": createPositionalPseudo(function( matchIndexes, length ) { + var i = 0; + for ( ; i < length; i += 2 ) { + matchIndexes.push( i ); + } + return matchIndexes; + }), + + "odd": createPositionalPseudo(function( matchIndexes, length ) { + var i = 1; + for ( ; i < length; i += 2 ) { + matchIndexes.push( i ); + } + return matchIndexes; + }), + + "lt": createPositionalPseudo(function( matchIndexes, length, argument ) { + var i = argument < 0 ? argument + length : argument; + for ( ; --i >= 0; ) { + matchIndexes.push( i ); + } + return matchIndexes; + }), + + "gt": createPositionalPseudo(function( matchIndexes, length, argument ) { + var i = argument < 0 ? argument + length : argument; + for ( ; ++i < length; ) { + matchIndexes.push( i ); + } + return matchIndexes; + }) + } +}; + +Expr.pseudos["nth"] = Expr.pseudos["eq"]; + +// Add button/input type pseudos +for ( i in { radio: true, checkbox: true, file: true, password: true, image: true } ) { + Expr.pseudos[ i ] = createInputPseudo( i ); +} +for ( i in { submit: true, reset: true } ) { + Expr.pseudos[ i ] = createButtonPseudo( i ); +} + +// Easy API for creating new setFilters +function setFilters() {} +setFilters.prototype = Expr.filters = Expr.pseudos; +Expr.setFilters = new setFilters(); + +function tokenize( selector, parseOnly ) { + var matched, match, tokens, type, + soFar, groups, preFilters, + cached = tokenCache[ selector + " " ]; + + if ( cached ) { + return parseOnly ? 0 : cached.slice( 0 ); + } + + soFar = selector; + groups = []; + preFilters = Expr.preFilter; + + while ( soFar ) { + + // Comma and first run + if ( !matched || (match = rcomma.exec( soFar )) ) { + if ( match ) { + // Don't consume trailing commas as valid + soFar = soFar.slice( match[0].length ) || soFar; + } + groups.push( (tokens = []) ); + } + + matched = false; + + // Combinators + if ( (match = rcombinators.exec( soFar )) ) { + matched = match.shift(); + tokens.push({ + value: matched, + // Cast descendant combinators to space + type: match[0].replace( rtrim, " " ) + }); + soFar = soFar.slice( matched.length ); + } + + // Filters + for ( type in Expr.filter ) { + if ( (match = matchExpr[ type ].exec( soFar )) && (!preFilters[ type ] || + (match = preFilters[ type ]( match ))) ) { + matched = match.shift(); + tokens.push({ + value: matched, + type: type, + matches: match + }); + soFar = soFar.slice( matched.length ); + } + } + + if ( !matched ) { + break; + } + } + + // Return the length of the invalid excess + // if we're just parsing + // Otherwise, throw an error or return tokens + return parseOnly ? + soFar.length : + soFar ? + Sizzle.error( selector ) : + // Cache the tokens + tokenCache( selector, groups ).slice( 0 ); +} + +function toSelector( tokens ) { + var i = 0, + len = tokens.length, + selector = ""; + for ( ; i < len; i++ ) { + selector += tokens[i].value; + } + return selector; +} + +function addCombinator( matcher, combinator, base ) { + var dir = combinator.dir, + checkNonElements = base && dir === "parentNode", + doneName = done++; + + return combinator.first ? + // Check against closest ancestor/preceding element + function( elem, context, xml ) { + while ( (elem = elem[ dir ]) ) { + if ( elem.nodeType === 1 || checkNonElements ) { + return matcher( elem, context, xml ); + } + } + } : + + // Check against all ancestor/preceding elements + function( elem, context, xml ) { + var oldCache, outerCache, + newCache = [ dirruns, doneName ]; + + // We can't set arbitrary data on XML nodes, so they don't benefit from dir caching + if ( xml ) { + while ( (elem = elem[ dir ]) ) { + if ( elem.nodeType === 1 || checkNonElements ) { + if ( matcher( elem, context, xml ) ) { + return true; + } + } + } + } else { + while ( (elem = elem[ dir ]) ) { + if ( elem.nodeType === 1 || checkNonElements ) { + outerCache = elem[ expando ] || (elem[ expando ] = {}); + if ( (oldCache = outerCache[ dir ]) && + oldCache[ 0 ] === dirruns && oldCache[ 1 ] === doneName ) { + + // Assign to newCache so results back-propagate to previous elements + return (newCache[ 2 ] = oldCache[ 2 ]); + } else { + // Reuse newcache so results back-propagate to previous elements + outerCache[ dir ] = newCache; + + // A match means we're done; a fail means we have to keep checking + if ( (newCache[ 2 ] = matcher( elem, context, xml )) ) { + return true; + } + } + } + } + } + }; +} + +function elementMatcher( matchers ) { + return matchers.length > 1 ? + function( elem, context, xml ) { + var i = matchers.length; + while ( i-- ) { + if ( !matchers[i]( elem, context, xml ) ) { + return false; + } + } + return true; + } : + matchers[0]; +} + +function condense( unmatched, map, filter, context, xml ) { + var elem, + newUnmatched = [], + i = 0, + len = unmatched.length, + mapped = map != null; + + for ( ; i < len; i++ ) { + if ( (elem = unmatched[i]) ) { + if ( !filter || filter( elem, context, xml ) ) { + newUnmatched.push( elem ); + if ( mapped ) { + map.push( i ); + } + } + } + } + + return newUnmatched; +} + +function setMatcher( preFilter, selector, matcher, postFilter, postFinder, postSelector ) { + if ( postFilter && !postFilter[ expando ] ) { + postFilter = setMatcher( postFilter ); + } + if ( postFinder && !postFinder[ expando ] ) { + postFinder = setMatcher( postFinder, postSelector ); + } + return markFunction(function( seed, results, context, xml ) { + var temp, i, elem, + preMap = [], + postMap = [], + preexisting = results.length, + + // Get initial elements from seed or context + elems = seed || multipleContexts( selector || "*", context.nodeType ? [ context ] : context, [] ), + + // Prefilter to get matcher input, preserving a map for seed-results synchronization + matcherIn = preFilter && ( seed || !selector ) ? + condense( elems, preMap, preFilter, context, xml ) : + elems, + + matcherOut = matcher ? + // If we have a postFinder, or filtered seed, or non-seed postFilter or preexisting results, + postFinder || ( seed ? preFilter : preexisting || postFilter ) ? + + // ...intermediate processing is necessary + [] : + + // ...otherwise use results directly + results : + matcherIn; + + // Find primary matches + if ( matcher ) { + matcher( matcherIn, matcherOut, context, xml ); + } + + // Apply postFilter + if ( postFilter ) { + temp = condense( matcherOut, postMap ); + postFilter( temp, [], context, xml ); + + // Un-match failing elements by moving them back to matcherIn + i = temp.length; + while ( i-- ) { + if ( (elem = temp[i]) ) { + matcherOut[ postMap[i] ] = !(matcherIn[ postMap[i] ] = elem); + } + } + } + + if ( seed ) { + if ( postFinder || preFilter ) { + if ( postFinder ) { + // Get the final matcherOut by condensing this intermediate into postFinder contexts + temp = []; + i = matcherOut.length; + while ( i-- ) { + if ( (elem = matcherOut[i]) ) { + // Restore matcherIn since elem is not yet a final match + temp.push( (matcherIn[i] = elem) ); + } + } + postFinder( null, (matcherOut = []), temp, xml ); + } + + // Move matched elements from seed to results to keep them synchronized + i = matcherOut.length; + while ( i-- ) { + if ( (elem = matcherOut[i]) && + (temp = postFinder ? indexOf.call( seed, elem ) : preMap[i]) > -1 ) { + + seed[temp] = !(results[temp] = elem); + } + } + } + + // Add elements to results, through postFinder if defined + } else { + matcherOut = condense( + matcherOut === results ? + matcherOut.splice( preexisting, matcherOut.length ) : + matcherOut + ); + if ( postFinder ) { + postFinder( null, results, matcherOut, xml ); + } else { + push.apply( results, matcherOut ); + } + } + }); +} + +function matcherFromTokens( tokens ) { + var checkContext, matcher, j, + len = tokens.length, + leadingRelative = Expr.relative[ tokens[0].type ], + implicitRelative = leadingRelative || Expr.relative[" "], + i = leadingRelative ? 1 : 0, + + // The foundational matcher ensures that elements are reachable from top-level context(s) + matchContext = addCombinator( function( elem ) { + return elem === checkContext; + }, implicitRelative, true ), + matchAnyContext = addCombinator( function( elem ) { + return indexOf.call( checkContext, elem ) > -1; + }, implicitRelative, true ), + matchers = [ function( elem, context, xml ) { + return ( !leadingRelative && ( xml || context !== outermostContext ) ) || ( + (checkContext = context).nodeType ? + matchContext( elem, context, xml ) : + matchAnyContext( elem, context, xml ) ); + } ]; + + for ( ; i < len; i++ ) { + if ( (matcher = Expr.relative[ tokens[i].type ]) ) { + matchers = [ addCombinator(elementMatcher( matchers ), matcher) ]; + } else { + matcher = Expr.filter[ tokens[i].type ].apply( null, tokens[i].matches ); + + // Return special upon seeing a positional matcher + if ( matcher[ expando ] ) { + // Find the next relative operator (if any) for proper handling + j = ++i; + for ( ; j < len; j++ ) { + if ( Expr.relative[ tokens[j].type ] ) { + break; + } + } + return setMatcher( + i > 1 && elementMatcher( matchers ), + i > 1 && toSelector( + // If the preceding token was a descendant combinator, insert an implicit any-element `*` + tokens.slice( 0, i - 1 ).concat({ value: tokens[ i - 2 ].type === " " ? "*" : "" }) + ).replace( rtrim, "$1" ), + matcher, + i < j && matcherFromTokens( tokens.slice( i, j ) ), + j < len && matcherFromTokens( (tokens = tokens.slice( j )) ), + j < len && toSelector( tokens ) + ); + } + matchers.push( matcher ); + } + } + + return elementMatcher( matchers ); +} + +function matcherFromGroupMatchers( elementMatchers, setMatchers ) { + var bySet = setMatchers.length > 0, + byElement = elementMatchers.length > 0, + superMatcher = function( seed, context, xml, results, outermost ) { + var elem, j, matcher, + matchedCount = 0, + i = "0", + unmatched = seed && [], + setMatched = [], + contextBackup = outermostContext, + // We must always have either seed elements or outermost context + elems = seed || byElement && Expr.find["TAG"]( "*", outermost ), + // Use integer dirruns iff this is the outermost matcher + dirrunsUnique = (dirruns += contextBackup == null ? 1 : Math.random() || 0.1), + len = elems.length; + + if ( outermost ) { + outermostContext = context !== document && context; + } + + // Add elements passing elementMatchers directly to results + // Keep `i` a string if there are no elements so `matchedCount` will be "00" below + // Support: IE<9, Safari + // Tolerate NodeList properties (IE: "length"; Safari: ) matching elements by id + for ( ; i !== len && (elem = elems[i]) != null; i++ ) { + if ( byElement && elem ) { + j = 0; + while ( (matcher = elementMatchers[j++]) ) { + if ( matcher( elem, context, xml ) ) { + results.push( elem ); + break; + } + } + if ( outermost ) { + dirruns = dirrunsUnique; + } + } + + // Track unmatched elements for set filters + if ( bySet ) { + // They will have gone through all possible matchers + if ( (elem = !matcher && elem) ) { + matchedCount--; + } + + // Lengthen the array for every element, matched or not + if ( seed ) { + unmatched.push( elem ); + } + } + } + + // Apply set filters to unmatched elements + matchedCount += i; + if ( bySet && i !== matchedCount ) { + j = 0; + while ( (matcher = setMatchers[j++]) ) { + matcher( unmatched, setMatched, context, xml ); + } + + if ( seed ) { + // Reintegrate element matches to eliminate the need for sorting + if ( matchedCount > 0 ) { + while ( i-- ) { + if ( !(unmatched[i] || setMatched[i]) ) { + setMatched[i] = pop.call( results ); + } + } + } + + // Discard index placeholder values to get only actual matches + setMatched = condense( setMatched ); + } + + // Add matches to results + push.apply( results, setMatched ); + + // Seedless set matches succeeding multiple successful matchers stipulate sorting + if ( outermost && !seed && setMatched.length > 0 && + ( matchedCount + setMatchers.length ) > 1 ) { + + Sizzle.uniqueSort( results ); + } + } + + // Override manipulation of globals by nested matchers + if ( outermost ) { + dirruns = dirrunsUnique; + outermostContext = contextBackup; + } + + return unmatched; + }; + + return bySet ? + markFunction( superMatcher ) : + superMatcher; +} + +compile = Sizzle.compile = function( selector, group /* Internal Use Only */ ) { + var i, + setMatchers = [], + elementMatchers = [], + cached = compilerCache[ selector + " " ]; + + if ( !cached ) { + // Generate a function of recursive functions that can be used to check each element + if ( !group ) { + group = tokenize( selector ); + } + i = group.length; + while ( i-- ) { + cached = matcherFromTokens( group[i] ); + if ( cached[ expando ] ) { + setMatchers.push( cached ); + } else { + elementMatchers.push( cached ); + } + } + + // Cache the compiled function + cached = compilerCache( selector, matcherFromGroupMatchers( elementMatchers, setMatchers ) ); + } + return cached; +}; + +function multipleContexts( selector, contexts, results ) { + var i = 0, + len = contexts.length; + for ( ; i < len; i++ ) { + Sizzle( selector, contexts[i], results ); + } + return results; +} + +function select( selector, context, results, seed ) { + var i, tokens, token, type, find, + match = tokenize( selector ); + + if ( !seed ) { + // Try to minimize operations if there is only one group + if ( match.length === 1 ) { + + // Take a shortcut and set the context if the root selector is an ID + tokens = match[0] = match[0].slice( 0 ); + if ( tokens.length > 2 && (token = tokens[0]).type === "ID" && + support.getById && context.nodeType === 9 && documentIsHTML && + Expr.relative[ tokens[1].type ] ) { + + context = ( Expr.find["ID"]( token.matches[0].replace(runescape, funescape), context ) || [] )[0]; + if ( !context ) { + return results; + } + selector = selector.slice( tokens.shift().value.length ); + } + + // Fetch a seed set for right-to-left matching + i = matchExpr["needsContext"].test( selector ) ? 0 : tokens.length; + while ( i-- ) { + token = tokens[i]; + + // Abort if we hit a combinator + if ( Expr.relative[ (type = token.type) ] ) { + break; + } + if ( (find = Expr.find[ type ]) ) { + // Search, expanding context for leading sibling combinators + if ( (seed = find( + token.matches[0].replace( runescape, funescape ), + rsibling.test( tokens[0].type ) && testContext( context.parentNode ) || context + )) ) { + + // If seed is empty or no tokens remain, we can return early + tokens.splice( i, 1 ); + selector = seed.length && toSelector( tokens ); + if ( !selector ) { + push.apply( results, seed ); + return results; + } + + break; + } + } + } + } + } + + // Compile and execute a filtering function + // Provide `match` to avoid retokenization if we modified the selector above + compile( selector, match )( + seed, + context, + !documentIsHTML, + results, + rsibling.test( selector ) && testContext( context.parentNode ) || context + ); + return results; +} + +// One-time assignments + +// Sort stability +support.sortStable = expando.split("").sort( sortOrder ).join("") === expando; + +// Support: Chrome<14 +// Always assume duplicates if they aren't passed to the comparison function +support.detectDuplicates = !!hasDuplicate; + +// Initialize against the default document +setDocument(); + +// Support: Webkit<537.32 - Safari 6.0.3/Chrome 25 (fixed in Chrome 27) +// Detached nodes confoundingly follow *each other* +support.sortDetached = assert(function( div1 ) { + // Should return 1, but returns 4 (following) + return div1.compareDocumentPosition( document.createElement("div") ) & 1; +}); + +// Support: IE<8 +// Prevent attribute/property "interpolation" +// http://msdn.microsoft.com/en-us/library/ms536429%28VS.85%29.aspx +if ( !assert(function( div ) { + div.innerHTML = "    "; + return div.firstChild.getAttribute("href") === "#" ; +}) ) { + addHandle( "type|href|height|width", function( elem, name, isXML ) { + if ( !isXML ) { + return elem.getAttribute( name, name.toLowerCase() === "type" ? 1 : 2 ); + } + }); +} + +// Support: IE<9 +// Use defaultValue in place of getAttribute("value") +if ( !support.attributes || !assert(function( div ) { + div.innerHTML = ""; + div.firstChild.setAttribute( "value", "" ); + return div.firstChild.getAttribute( "value" ) === ""; +}) ) { + addHandle( "value", function( elem, name, isXML ) { + if ( !isXML && elem.nodeName.toLowerCase() === "input" ) { + return elem.defaultValue; + } + }); +} + +// Support: IE<9 +// Use getAttributeNode to fetch booleans when getAttribute lies +if ( !assert(function( div ) { + return div.getAttribute("disabled") == null; +}) ) { + addHandle( booleans, function( elem, name, isXML ) { + var val; + if ( !isXML ) { + return elem[ name ] === true ? name.toLowerCase() : + (val = elem.getAttributeNode( name )) && val.specified ? + val.value : + null; + } + }); +} + +return Sizzle; + +})( window ); + + + +jQuery.find = Sizzle; +jQuery.expr = Sizzle.selectors; +jQuery.expr[":"] = jQuery.expr.pseudos; +jQuery.unique = Sizzle.uniqueSort; +jQuery.text = Sizzle.getText; +jQuery.isXMLDoc = Sizzle.isXML; +jQuery.contains = Sizzle.contains; + + + +var rneedsContext = jQuery.expr.match.needsContext; + +var rsingleTag = (/^<(\w+)\s*\/?>(?:<\/\1>|)$/); + + + +var risSimple = /^.[^:#\[\.,]*$/; + +// Implement the identical functionality for filter and not +function winnow( elements, qualifier, not ) { + if ( jQuery.isFunction( qualifier ) ) { + return jQuery.grep( elements, function( elem, i ) { + /* jshint -W018 */ + return !!qualifier.call( elem, i, elem ) !== not; + }); + + } + + if ( qualifier.nodeType ) { + return jQuery.grep( elements, function( elem ) { + return ( elem === qualifier ) !== not; + }); + + } + + if ( typeof qualifier === "string" ) { + if ( risSimple.test( qualifier ) ) { + return jQuery.filter( qualifier, elements, not ); + } + + qualifier = jQuery.filter( qualifier, elements ); + } + + return jQuery.grep( elements, function( elem ) { + return ( jQuery.inArray( elem, qualifier ) >= 0 ) !== not; + }); +} + +jQuery.filter = function( expr, elems, not ) { + var elem = elems[ 0 ]; + + if ( not ) { + expr = ":not(" + expr + ")"; + } + + return elems.length === 1 && elem.nodeType === 1 ? + jQuery.find.matchesSelector( elem, expr ) ? [ elem ] : [] : + jQuery.find.matches( expr, jQuery.grep( elems, function( elem ) { + return elem.nodeType === 1; + })); +}; + +jQuery.fn.extend({ + find: function( selector ) { + var i, + ret = [], + self = this, + len = self.length; + + if ( typeof selector !== "string" ) { + return this.pushStack( jQuery( selector ).filter(function() { + for ( i = 0; i < len; i++ ) { + if ( jQuery.contains( self[ i ], this ) ) { + return true; + } + } + }) ); + } + + for ( i = 0; i < len; i++ ) { + jQuery.find( selector, self[ i ], ret ); + } + + // Needed because $( selector, context ) becomes $( context ).find( selector ) + ret = this.pushStack( len > 1 ? jQuery.unique( ret ) : ret ); + ret.selector = this.selector ? this.selector + " " + selector : selector; + return ret; + }, + filter: function( selector ) { + return this.pushStack( winnow(this, selector || [], false) ); + }, + not: function( selector ) { + return this.pushStack( winnow(this, selector || [], true) ); + }, + is: function( selector ) { + return !!winnow( + this, + + // If this is a positional/relative selector, check membership in the returned set + // so $("p:first").is("p:last") won't return true for a doc with two "p". + typeof selector === "string" && rneedsContext.test( selector ) ? + jQuery( selector ) : + selector || [], + false + ).length; + } +}); + + +// Initialize a jQuery object + + +// A central reference to the root jQuery(document) +var rootjQuery, + + // Use the correct document accordingly with window argument (sandbox) + document = window.document, + + // A simple way to check for HTML strings + // Prioritize #id over to avoid XSS via location.hash (#9521) + // Strict HTML recognition (#11290: must start with <) + rquickExpr = /^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/, + + init = jQuery.fn.init = function( selector, context ) { + var match, elem; + + // HANDLE: $(""), $(null), $(undefined), $(false) + if ( !selector ) { + return this; + } + + // Handle HTML strings + if ( typeof selector === "string" ) { + if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) { + // Assume that strings that start and end with <> are HTML and skip the regex check + match = [ null, selector, null ]; + + } else { + match = rquickExpr.exec( selector ); + } + + // Match html or make sure no context is specified for #id + if ( match && (match[1] || !context) ) { + + // HANDLE: $(html) -> $(array) + if ( match[1] ) { + context = context instanceof jQuery ? context[0] : context; + + // scripts is true for back-compat + // Intentionally let the error be thrown if parseHTML is not present + jQuery.merge( this, jQuery.parseHTML( + match[1], + context && context.nodeType ? context.ownerDocument || context : document, + true + ) ); + + // HANDLE: $(html, props) + if ( rsingleTag.test( match[1] ) && jQuery.isPlainObject( context ) ) { + for ( match in context ) { + // Properties of context are called as methods if possible + if ( jQuery.isFunction( this[ match ] ) ) { + this[ match ]( context[ match ] ); + + // ...and otherwise set as attributes + } else { + this.attr( match, context[ match ] ); + } + } + } + + return this; + + // HANDLE: $(#id) + } else { + elem = document.getElementById( match[2] ); + + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + if ( elem && elem.parentNode ) { + // Handle the case where IE and Opera return items + // by name instead of ID + if ( elem.id !== match[2] ) { + return rootjQuery.find( selector ); + } + + // Otherwise, we inject the element directly into the jQuery object + this.length = 1; + this[0] = elem; + } + + this.context = document; + this.selector = selector; + return this; + } + + // HANDLE: $(expr, $(...)) + } else if ( !context || context.jquery ) { + return ( context || rootjQuery ).find( selector ); + + // HANDLE: $(expr, context) + // (which is just equivalent to: $(context).find(expr) + } else { + return this.constructor( context ).find( selector ); + } + + // HANDLE: $(DOMElement) + } else if ( selector.nodeType ) { + this.context = this[0] = selector; + this.length = 1; + return this; + + // HANDLE: $(function) + // Shortcut for document ready + } else if ( jQuery.isFunction( selector ) ) { + return typeof rootjQuery.ready !== "undefined" ? + rootjQuery.ready( selector ) : + // Execute immediately if ready is not present + selector( jQuery ); + } + + if ( selector.selector !== undefined ) { + this.selector = selector.selector; + this.context = selector.context; + } + + return jQuery.makeArray( selector, this ); + }; + +// Give the init function the jQuery prototype for later instantiation +init.prototype = jQuery.fn; + +// Initialize central reference +rootjQuery = jQuery( document ); + + +var rparentsprev = /^(?:parents|prev(?:Until|All))/, + // methods guaranteed to produce a unique set when starting from a unique set + guaranteedUnique = { + children: true, + contents: true, + next: true, + prev: true + }; + +jQuery.extend({ + dir: function( elem, dir, until ) { + var matched = [], + cur = elem[ dir ]; + + while ( cur && cur.nodeType !== 9 && (until === undefined || cur.nodeType !== 1 || !jQuery( cur ).is( until )) ) { + if ( cur.nodeType === 1 ) { + matched.push( cur ); + } + cur = cur[dir]; + } + return matched; + }, + + sibling: function( n, elem ) { + var r = []; + + for ( ; n; n = n.nextSibling ) { + if ( n.nodeType === 1 && n !== elem ) { + r.push( n ); + } + } + + return r; + } +}); + +jQuery.fn.extend({ + has: function( target ) { + var i, + targets = jQuery( target, this ), + len = targets.length; + + return this.filter(function() { + for ( i = 0; i < len; i++ ) { + if ( jQuery.contains( this, targets[i] ) ) { + return true; + } + } + }); + }, + + closest: function( selectors, context ) { + var cur, + i = 0, + l = this.length, + matched = [], + pos = rneedsContext.test( selectors ) || typeof selectors !== "string" ? + jQuery( selectors, context || this.context ) : + 0; + + for ( ; i < l; i++ ) { + for ( cur = this[i]; cur && cur !== context; cur = cur.parentNode ) { + // Always skip document fragments + if ( cur.nodeType < 11 && (pos ? + pos.index(cur) > -1 : + + // Don't pass non-elements to Sizzle + cur.nodeType === 1 && + jQuery.find.matchesSelector(cur, selectors)) ) { + + matched.push( cur ); + break; + } + } + } + + return this.pushStack( matched.length > 1 ? jQuery.unique( matched ) : matched ); + }, + + // Determine the position of an element within + // the matched set of elements + index: function( elem ) { + + // No argument, return index in parent + if ( !elem ) { + return ( this[0] && this[0].parentNode ) ? this.first().prevAll().length : -1; + } + + // index in selector + if ( typeof elem === "string" ) { + return jQuery.inArray( this[0], jQuery( elem ) ); + } + + // Locate the position of the desired element + return jQuery.inArray( + // If it receives a jQuery object, the first element is used + elem.jquery ? elem[0] : elem, this ); + }, + + add: function( selector, context ) { + return this.pushStack( + jQuery.unique( + jQuery.merge( this.get(), jQuery( selector, context ) ) + ) + ); + }, + + addBack: function( selector ) { + return this.add( selector == null ? + this.prevObject : this.prevObject.filter(selector) + ); + } +}); + +function sibling( cur, dir ) { + do { + cur = cur[ dir ]; + } while ( cur && cur.nodeType !== 1 ); + + return cur; +} + +jQuery.each({ + parent: function( elem ) { + var parent = elem.parentNode; + return parent && parent.nodeType !== 11 ? parent : null; + }, + parents: function( elem ) { + return jQuery.dir( elem, "parentNode" ); + }, + parentsUntil: function( elem, i, until ) { + return jQuery.dir( elem, "parentNode", until ); + }, + next: function( elem ) { + return sibling( elem, "nextSibling" ); + }, + prev: function( elem ) { + return sibling( elem, "previousSibling" ); + }, + nextAll: function( elem ) { + return jQuery.dir( elem, "nextSibling" ); + }, + prevAll: function( elem ) { + return jQuery.dir( elem, "previousSibling" ); + }, + nextUntil: function( elem, i, until ) { + return jQuery.dir( elem, "nextSibling", until ); + }, + prevUntil: function( elem, i, until ) { + return jQuery.dir( elem, "previousSibling", until ); + }, + siblings: function( elem ) { + return jQuery.sibling( ( elem.parentNode || {} ).firstChild, elem ); + }, + children: function( elem ) { + return jQuery.sibling( elem.firstChild ); + }, + contents: function( elem ) { + return jQuery.nodeName( elem, "iframe" ) ? + elem.contentDocument || elem.contentWindow.document : + jQuery.merge( [], elem.childNodes ); + } +}, function( name, fn ) { + jQuery.fn[ name ] = function( until, selector ) { + var ret = jQuery.map( this, fn, until ); + + if ( name.slice( -5 ) !== "Until" ) { + selector = until; + } + + if ( selector && typeof selector === "string" ) { + ret = jQuery.filter( selector, ret ); + } + + if ( this.length > 1 ) { + // Remove duplicates + if ( !guaranteedUnique[ name ] ) { + ret = jQuery.unique( ret ); + } + + // Reverse order for parents* and prev-derivatives + if ( rparentsprev.test( name ) ) { + ret = ret.reverse(); + } + } + + return this.pushStack( ret ); + }; +}); +var rnotwhite = (/\S+/g); + + + +// String to Object options format cache +var optionsCache = {}; + +// Convert String-formatted options into Object-formatted ones and store in cache +function createOptions( options ) { + var object = optionsCache[ options ] = {}; + jQuery.each( options.match( rnotwhite ) || [], function( _, flag ) { + object[ flag ] = true; + }); + return object; +} + +/* + * Create a callback list using the following parameters: + * + * options: an optional list of space-separated options that will change how + * the callback list behaves or a more traditional option object + * + * By default a callback list will act like an event callback list and can be + * "fired" multiple times. + * + * Possible options: + * + * once: will ensure the callback list can only be fired once (like a Deferred) + * + * memory: will keep track of previous values and will call any callback added + * after the list has been fired right away with the latest "memorized" + * values (like a Deferred) + * + * unique: will ensure a callback can only be added once (no duplicate in the list) + * + * stopOnFalse: interrupt callings when a callback returns false + * + */ +jQuery.Callbacks = function( options ) { + + // Convert options from String-formatted to Object-formatted if needed + // (we check in cache first) + options = typeof options === "string" ? + ( optionsCache[ options ] || createOptions( options ) ) : + jQuery.extend( {}, options ); + + var // Flag to know if list is currently firing + firing, + // Last fire value (for non-forgettable lists) + memory, + // Flag to know if list was already fired + fired, + // End of the loop when firing + firingLength, + // Index of currently firing callback (modified by remove if needed) + firingIndex, + // First callback to fire (used internally by add and fireWith) + firingStart, + // Actual callback list + list = [], + // Stack of fire calls for repeatable lists + stack = !options.once && [], + // Fire callbacks + fire = function( data ) { + memory = options.memory && data; + fired = true; + firingIndex = firingStart || 0; + firingStart = 0; + firingLength = list.length; + firing = true; + for ( ; list && firingIndex < firingLength; firingIndex++ ) { + if ( list[ firingIndex ].apply( data[ 0 ], data[ 1 ] ) === false && options.stopOnFalse ) { + memory = false; // To prevent further calls using add + break; + } + } + firing = false; + if ( list ) { + if ( stack ) { + if ( stack.length ) { + fire( stack.shift() ); + } + } else if ( memory ) { + list = []; + } else { + self.disable(); + } + } + }, + // Actual Callbacks object + self = { + // Add a callback or a collection of callbacks to the list + add: function() { + if ( list ) { + // First, we save the current length + var start = list.length; + (function add( args ) { + jQuery.each( args, function( _, arg ) { + var type = jQuery.type( arg ); + if ( type === "function" ) { + if ( !options.unique || !self.has( arg ) ) { + list.push( arg ); + } + } else if ( arg && arg.length && type !== "string" ) { + // Inspect recursively + add( arg ); + } + }); + })( arguments ); + // Do we need to add the callbacks to the + // current firing batch? + if ( firing ) { + firingLength = list.length; + // With memory, if we're not firing then + // we should call right away + } else if ( memory ) { + firingStart = start; + fire( memory ); + } + } + return this; + }, + // Remove a callback from the list + remove: function() { + if ( list ) { + jQuery.each( arguments, function( _, arg ) { + var index; + while ( ( index = jQuery.inArray( arg, list, index ) ) > -1 ) { + list.splice( index, 1 ); + // Handle firing indexes + if ( firing ) { + if ( index <= firingLength ) { + firingLength--; + } + if ( index <= firingIndex ) { + firingIndex--; + } + } + } + }); + } + return this; + }, + // Check if a given callback is in the list. + // If no argument is given, return whether or not list has callbacks attached. + has: function( fn ) { + return fn ? jQuery.inArray( fn, list ) > -1 : !!( list && list.length ); + }, + // Remove all callbacks from the list + empty: function() { + list = []; + firingLength = 0; + return this; + }, + // Have the list do nothing anymore + disable: function() { + list = stack = memory = undefined; + return this; + }, + // Is it disabled? + disabled: function() { + return !list; + }, + // Lock the list in its current state + lock: function() { + stack = undefined; + if ( !memory ) { + self.disable(); + } + return this; + }, + // Is it locked? + locked: function() { + return !stack; + }, + // Call all callbacks with the given context and arguments + fireWith: function( context, args ) { + if ( list && ( !fired || stack ) ) { + args = args || []; + args = [ context, args.slice ? args.slice() : args ]; + if ( firing ) { + stack.push( args ); + } else { + fire( args ); + } + } + return this; + }, + // Call all the callbacks with the given arguments + fire: function() { + self.fireWith( this, arguments ); + return this; + }, + // To know if the callbacks have already been called at least once + fired: function() { + return !!fired; + } + }; + + return self; +}; + + +jQuery.extend({ + + Deferred: function( func ) { + var tuples = [ + // action, add listener, listener list, final state + [ "resolve", "done", jQuery.Callbacks("once memory"), "resolved" ], + [ "reject", "fail", jQuery.Callbacks("once memory"), "rejected" ], + [ "notify", "progress", jQuery.Callbacks("memory") ] + ], + state = "pending", + promise = { + state: function() { + return state; + }, + always: function() { + deferred.done( arguments ).fail( arguments ); + return this; + }, + then: function( /* fnDone, fnFail, fnProgress */ ) { + var fns = arguments; + return jQuery.Deferred(function( newDefer ) { + jQuery.each( tuples, function( i, tuple ) { + var fn = jQuery.isFunction( fns[ i ] ) && fns[ i ]; + // deferred[ done | fail | progress ] for forwarding actions to newDefer + deferred[ tuple[1] ](function() { + var returned = fn && fn.apply( this, arguments ); + if ( returned && jQuery.isFunction( returned.promise ) ) { + returned.promise() + .done( newDefer.resolve ) + .fail( newDefer.reject ) + .progress( newDefer.notify ); + } else { + newDefer[ tuple[ 0 ] + "With" ]( this === promise ? newDefer.promise() : this, fn ? [ returned ] : arguments ); + } + }); + }); + fns = null; + }).promise(); + }, + // Get a promise for this deferred + // If obj is provided, the promise aspect is added to the object + promise: function( obj ) { + return obj != null ? jQuery.extend( obj, promise ) : promise; + } + }, + deferred = {}; + + // Keep pipe for back-compat + promise.pipe = promise.then; + + // Add list-specific methods + jQuery.each( tuples, function( i, tuple ) { + var list = tuple[ 2 ], + stateString = tuple[ 3 ]; + + // promise[ done | fail | progress ] = list.add + promise[ tuple[1] ] = list.add; + + // Handle state + if ( stateString ) { + list.add(function() { + // state = [ resolved | rejected ] + state = stateString; + + // [ reject_list | resolve_list ].disable; progress_list.lock + }, tuples[ i ^ 1 ][ 2 ].disable, tuples[ 2 ][ 2 ].lock ); + } + + // deferred[ resolve | reject | notify ] + deferred[ tuple[0] ] = function() { + deferred[ tuple[0] + "With" ]( this === deferred ? promise : this, arguments ); + return this; + }; + deferred[ tuple[0] + "With" ] = list.fireWith; + }); + + // Make the deferred a promise + promise.promise( deferred ); + + // Call given func if any + if ( func ) { + func.call( deferred, deferred ); + } + + // All done! + return deferred; + }, + + // Deferred helper + when: function( subordinate /* , ..., subordinateN */ ) { + var i = 0, + resolveValues = slice.call( arguments ), + length = resolveValues.length, + + // the count of uncompleted subordinates + remaining = length !== 1 || ( subordinate && jQuery.isFunction( subordinate.promise ) ) ? length : 0, + + // the master Deferred. If resolveValues consist of only a single Deferred, just use that. + deferred = remaining === 1 ? subordinate : jQuery.Deferred(), + + // Update function for both resolve and progress values + updateFunc = function( i, contexts, values ) { + return function( value ) { + contexts[ i ] = this; + values[ i ] = arguments.length > 1 ? slice.call( arguments ) : value; + if ( values === progressValues ) { + deferred.notifyWith( contexts, values ); + + } else if ( !(--remaining) ) { + deferred.resolveWith( contexts, values ); + } + }; + }, + + progressValues, progressContexts, resolveContexts; + + // add listeners to Deferred subordinates; treat others as resolved + if ( length > 1 ) { + progressValues = new Array( length ); + progressContexts = new Array( length ); + resolveContexts = new Array( length ); + for ( ; i < length; i++ ) { + if ( resolveValues[ i ] && jQuery.isFunction( resolveValues[ i ].promise ) ) { + resolveValues[ i ].promise() + .done( updateFunc( i, resolveContexts, resolveValues ) ) + .fail( deferred.reject ) + .progress( updateFunc( i, progressContexts, progressValues ) ); + } else { + --remaining; + } + } + } + + // if we're not waiting on anything, resolve the master + if ( !remaining ) { + deferred.resolveWith( resolveContexts, resolveValues ); + } + + return deferred.promise(); + } +}); + + +// The deferred used on DOM ready +var readyList; + +jQuery.fn.ready = function( fn ) { + // Add the callback + jQuery.ready.promise().done( fn ); + + return this; +}; + +jQuery.extend({ + // Is the DOM ready to be used? Set to true once it occurs. + isReady: false, + + // A counter to track how many items to wait for before + // the ready event fires. See #6781 + readyWait: 1, + + // Hold (or release) the ready event + holdReady: function( hold ) { + if ( hold ) { + jQuery.readyWait++; + } else { + jQuery.ready( true ); + } + }, + + // Handle when the DOM is ready + ready: function( wait ) { + + // Abort if there are pending holds or we're already ready + if ( wait === true ? --jQuery.readyWait : jQuery.isReady ) { + return; + } + + // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443). + if ( !document.body ) { + return setTimeout( jQuery.ready ); + } + + // Remember that the DOM is ready + jQuery.isReady = true; + + // If a normal DOM Ready event fired, decrement, and wait if need be + if ( wait !== true && --jQuery.readyWait > 0 ) { + return; + } + + // If there are functions bound, to execute + readyList.resolveWith( document, [ jQuery ] ); + + // Trigger any bound ready events + if ( jQuery.fn.trigger ) { + jQuery( document ).trigger("ready").off("ready"); + } + } +}); + +/** + * Clean-up method for dom ready events + */ +function detach() { + if ( document.addEventListener ) { + document.removeEventListener( "DOMContentLoaded", completed, false ); + window.removeEventListener( "load", completed, false ); + + } else { + document.detachEvent( "onreadystatechange", completed ); + window.detachEvent( "onload", completed ); + } +} + +/** + * The ready event handler and self cleanup method + */ +function completed() { + // readyState === "complete" is good enough for us to call the dom ready in oldIE + if ( document.addEventListener || event.type === "load" || document.readyState === "complete" ) { + detach(); + jQuery.ready(); + } +} + +jQuery.ready.promise = function( obj ) { + if ( !readyList ) { + + readyList = jQuery.Deferred(); + + // Catch cases where $(document).ready() is called after the browser event has already occurred. + // we once tried to use readyState "interactive" here, but it caused issues like the one + // discovered by ChrisS here: http://bugs.jquery.com/ticket/12282#comment:15 + if ( document.readyState === "complete" ) { + // Handle it asynchronously to allow scripts the opportunity to delay ready + setTimeout( jQuery.ready ); + + // Standards-based browsers support DOMContentLoaded + } else if ( document.addEventListener ) { + // Use the handy event callback + document.addEventListener( "DOMContentLoaded", completed, false ); + + // A fallback to window.onload, that will always work + window.addEventListener( "load", completed, false ); + + // If IE event model is used + } else { + // Ensure firing before onload, maybe late but safe also for iframes + document.attachEvent( "onreadystatechange", completed ); + + // A fallback to window.onload, that will always work + window.attachEvent( "onload", completed ); + + // If IE and not a frame + // continually check to see if the document is ready + var top = false; + + try { + top = window.frameElement == null && document.documentElement; + } catch(e) {} + + if ( top && top.doScroll ) { + (function doScrollCheck() { + if ( !jQuery.isReady ) { + + try { + // Use the trick by Diego Perini + // http://javascript.nwbox.com/IEContentLoaded/ + top.doScroll("left"); + } catch(e) { + return setTimeout( doScrollCheck, 50 ); + } + + // detach all dom ready events + detach(); + + // and execute any waiting functions + jQuery.ready(); + } + })(); + } + } + } + return readyList.promise( obj ); +}; + + +var strundefined = typeof undefined; + + + +// Support: IE<9 +// Iteration over object's inherited properties before its own +var i; +for ( i in jQuery( support ) ) { + break; +} +support.ownLast = i !== "0"; + +// Note: most support tests are defined in their respective modules. +// false until the test is run +support.inlineBlockNeedsLayout = false; + +jQuery(function() { + // We need to execute this one support test ASAP because we need to know + // if body.style.zoom needs to be set. + + var container, div, + body = document.getElementsByTagName("body")[0]; + + if ( !body ) { + // Return for frameset docs that don't have a body + return; + } + + // Setup + container = document.createElement( "div" ); + container.style.cssText = "border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px"; + + div = document.createElement( "div" ); + body.appendChild( container ).appendChild( div ); + + if ( typeof div.style.zoom !== strundefined ) { + // Support: IE<8 + // Check if natively block-level elements act like inline-block + // elements when setting their display to 'inline' and giving + // them layout + div.style.cssText = "border:0;margin:0;width:1px;padding:1px;display:inline;zoom:1"; + + if ( (support.inlineBlockNeedsLayout = ( div.offsetWidth === 3 )) ) { + // Prevent IE 6 from affecting layout for positioned elements #11048 + // Prevent IE from shrinking the body in IE 7 mode #12869 + // Support: IE<8 + body.style.zoom = 1; + } + } + + body.removeChild( container ); + + // Null elements to avoid leaks in IE + container = div = null; +}); + + + + +(function() { + var div = document.createElement( "div" ); + + // Execute the test only if not already executed in another module. + if (support.deleteExpando == null) { + // Support: IE<9 + support.deleteExpando = true; + try { + delete div.test; + } catch( e ) { + support.deleteExpando = false; + } + } + + // Null elements to avoid leaks in IE. + div = null; +})(); + + +/** + * Determines whether an object can have data + */ +jQuery.acceptData = function( elem ) { + var noData = jQuery.noData[ (elem.nodeName + " ").toLowerCase() ], + nodeType = +elem.nodeType || 1; + + // Do not set data on non-element DOM nodes because it will not be cleared (#8335). + return nodeType !== 1 && nodeType !== 9 ? + false : + + // Nodes accept data unless otherwise specified; rejection can be conditional + !noData || noData !== true && elem.getAttribute("classid") === noData; +}; + + +var rbrace = /^(?:\{[\w\W]*\}|\[[\w\W]*\])$/, + rmultiDash = /([A-Z])/g; + +function dataAttr( elem, key, data ) { + // If nothing was found internally, try to fetch any + // data from the HTML5 data-* attribute + if ( data === undefined && elem.nodeType === 1 ) { + + var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase(); + + data = elem.getAttribute( name ); + + if ( typeof data === "string" ) { + try { + data = data === "true" ? true : + data === "false" ? false : + data === "null" ? null : + // Only convert to a number if it doesn't change the string + +data + "" === data ? +data : + rbrace.test( data ) ? jQuery.parseJSON( data ) : + data; + } catch( e ) {} + + // Make sure we set the data so it isn't changed later + jQuery.data( elem, key, data ); + + } else { + data = undefined; + } + } + + return data; +} + +// checks a cache object for emptiness +function isEmptyDataObject( obj ) { + var name; + for ( name in obj ) { + + // if the public data object is empty, the private is still empty + if ( name === "data" && jQuery.isEmptyObject( obj[name] ) ) { + continue; + } + if ( name !== "toJSON" ) { + return false; + } + } + + return true; +} + +function internalData( elem, name, data, pvt /* Internal Use Only */ ) { + if ( !jQuery.acceptData( elem ) ) { + return; + } + + var ret, thisCache, + internalKey = jQuery.expando, + + // We have to handle DOM nodes and JS objects differently because IE6-7 + // can't GC object references properly across the DOM-JS boundary + isNode = elem.nodeType, + + // Only DOM nodes need the global jQuery cache; JS object data is + // attached directly to the object so GC can occur automatically + cache = isNode ? jQuery.cache : elem, + + // Only defining an ID for JS objects if its cache already exists allows + // the code to shortcut on the same path as a DOM node with no cache + id = isNode ? elem[ internalKey ] : elem[ internalKey ] && internalKey; + + // Avoid doing any more work than we need to when trying to get data on an + // object that has no data at all + if ( (!id || !cache[id] || (!pvt && !cache[id].data)) && data === undefined && typeof name === "string" ) { + return; + } + + if ( !id ) { + // Only DOM nodes need a new unique ID for each element since their data + // ends up in the global cache + if ( isNode ) { + id = elem[ internalKey ] = deletedIds.pop() || jQuery.guid++; + } else { + id = internalKey; + } + } + + if ( !cache[ id ] ) { + // Avoid exposing jQuery metadata on plain JS objects when the object + // is serialized using JSON.stringify + cache[ id ] = isNode ? {} : { toJSON: jQuery.noop }; + } + + // An object can be passed to jQuery.data instead of a key/value pair; this gets + // shallow copied over onto the existing cache + if ( typeof name === "object" || typeof name === "function" ) { + if ( pvt ) { + cache[ id ] = jQuery.extend( cache[ id ], name ); + } else { + cache[ id ].data = jQuery.extend( cache[ id ].data, name ); + } + } + + thisCache = cache[ id ]; + + // jQuery data() is stored in a separate object inside the object's internal data + // cache in order to avoid key collisions between internal data and user-defined + // data. + if ( !pvt ) { + if ( !thisCache.data ) { + thisCache.data = {}; + } + + thisCache = thisCache.data; + } + + if ( data !== undefined ) { + thisCache[ jQuery.camelCase( name ) ] = data; + } + + // Check for both converted-to-camel and non-converted data property names + // If a data property was specified + if ( typeof name === "string" ) { + + // First Try to find as-is property data + ret = thisCache[ name ]; + + // Test for null|undefined property data + if ( ret == null ) { + + // Try to find the camelCased property + ret = thisCache[ jQuery.camelCase( name ) ]; + } + } else { + ret = thisCache; + } + + return ret; +} + +function internalRemoveData( elem, name, pvt ) { + if ( !jQuery.acceptData( elem ) ) { + return; + } + + var thisCache, i, + isNode = elem.nodeType, + + // See jQuery.data for more information + cache = isNode ? jQuery.cache : elem, + id = isNode ? elem[ jQuery.expando ] : jQuery.expando; + + // If there is already no cache entry for this object, there is no + // purpose in continuing + if ( !cache[ id ] ) { + return; + } + + if ( name ) { + + thisCache = pvt ? cache[ id ] : cache[ id ].data; + + if ( thisCache ) { + + // Support array or space separated string names for data keys + if ( !jQuery.isArray( name ) ) { + + // try the string as a key before any manipulation + if ( name in thisCache ) { + name = [ name ]; + } else { + + // split the camel cased version by spaces unless a key with the spaces exists + name = jQuery.camelCase( name ); + if ( name in thisCache ) { + name = [ name ]; + } else { + name = name.split(" "); + } + } + } else { + // If "name" is an array of keys... + // When data is initially created, via ("key", "val") signature, + // keys will be converted to camelCase. + // Since there is no way to tell _how_ a key was added, remove + // both plain key and camelCase key. #12786 + // This will only penalize the array argument path. + name = name.concat( jQuery.map( name, jQuery.camelCase ) ); + } + + i = name.length; + while ( i-- ) { + delete thisCache[ name[i] ]; + } + + // If there is no data left in the cache, we want to continue + // and let the cache object itself get destroyed + if ( pvt ? !isEmptyDataObject(thisCache) : !jQuery.isEmptyObject(thisCache) ) { + return; + } + } + } + + // See jQuery.data for more information + if ( !pvt ) { + delete cache[ id ].data; + + // Don't destroy the parent cache unless the internal data object + // had been the only thing left in it + if ( !isEmptyDataObject( cache[ id ] ) ) { + return; + } + } + + // Destroy the cache + if ( isNode ) { + jQuery.cleanData( [ elem ], true ); + + // Use delete when supported for expandos or `cache` is not a window per isWindow (#10080) + /* jshint eqeqeq: false */ + } else if ( support.deleteExpando || cache != cache.window ) { + /* jshint eqeqeq: true */ + delete cache[ id ]; + + // When all else fails, null + } else { + cache[ id ] = null; + } +} + +jQuery.extend({ + cache: {}, + + // The following elements (space-suffixed to avoid Object.prototype collisions) + // throw uncatchable exceptions if you attempt to set expando properties + noData: { + "applet ": true, + "embed ": true, + // ...but Flash objects (which have this classid) *can* handle expandos + "object ": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000" + }, + + hasData: function( elem ) { + elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ]; + return !!elem && !isEmptyDataObject( elem ); + }, + + data: function( elem, name, data ) { + return internalData( elem, name, data ); + }, + + removeData: function( elem, name ) { + return internalRemoveData( elem, name ); + }, + + // For internal use only. + _data: function( elem, name, data ) { + return internalData( elem, name, data, true ); + }, + + _removeData: function( elem, name ) { + return internalRemoveData( elem, name, true ); + } +}); + +jQuery.fn.extend({ + data: function( key, value ) { + var i, name, data, + elem = this[0], + attrs = elem && elem.attributes; + + // Special expections of .data basically thwart jQuery.access, + // so implement the relevant behavior ourselves + + // Gets all values + if ( key === undefined ) { + if ( this.length ) { + data = jQuery.data( elem ); + + if ( elem.nodeType === 1 && !jQuery._data( elem, "parsedAttrs" ) ) { + i = attrs.length; + while ( i-- ) { + name = attrs[i].name; + + if ( name.indexOf("data-") === 0 ) { + name = jQuery.camelCase( name.slice(5) ); + + dataAttr( elem, name, data[ name ] ); + } + } + jQuery._data( elem, "parsedAttrs", true ); + } + } + + return data; + } + + // Sets multiple values + if ( typeof key === "object" ) { + return this.each(function() { + jQuery.data( this, key ); + }); + } + + return arguments.length > 1 ? + + // Sets one value + this.each(function() { + jQuery.data( this, key, value ); + }) : + + // Gets one value + // Try to fetch any internally stored data first + elem ? dataAttr( elem, key, jQuery.data( elem, key ) ) : undefined; + }, + + removeData: function( key ) { + return this.each(function() { + jQuery.removeData( this, key ); + }); + } +}); + + +jQuery.extend({ + queue: function( elem, type, data ) { + var queue; + + if ( elem ) { + type = ( type || "fx" ) + "queue"; + queue = jQuery._data( elem, type ); + + // Speed up dequeue by getting out quickly if this is just a lookup + if ( data ) { + if ( !queue || jQuery.isArray(data) ) { + queue = jQuery._data( elem, type, jQuery.makeArray(data) ); + } else { + queue.push( data ); + } + } + return queue || []; + } + }, + + dequeue: function( elem, type ) { + type = type || "fx"; + + var queue = jQuery.queue( elem, type ), + startLength = queue.length, + fn = queue.shift(), + hooks = jQuery._queueHooks( elem, type ), + next = function() { + jQuery.dequeue( elem, type ); + }; + + // If the fx queue is dequeued, always remove the progress sentinel + if ( fn === "inprogress" ) { + fn = queue.shift(); + startLength--; + } + + if ( fn ) { + + // Add a progress sentinel to prevent the fx queue from being + // automatically dequeued + if ( type === "fx" ) { + queue.unshift( "inprogress" ); + } + + // clear up the last queue stop function + delete hooks.stop; + fn.call( elem, next, hooks ); + } + + if ( !startLength && hooks ) { + hooks.empty.fire(); + } + }, + + // not intended for public consumption - generates a queueHooks object, or returns the current one + _queueHooks: function( elem, type ) { + var key = type + "queueHooks"; + return jQuery._data( elem, key ) || jQuery._data( elem, key, { + empty: jQuery.Callbacks("once memory").add(function() { + jQuery._removeData( elem, type + "queue" ); + jQuery._removeData( elem, key ); + }) + }); + } +}); + +jQuery.fn.extend({ + queue: function( type, data ) { + var setter = 2; + + if ( typeof type !== "string" ) { + data = type; + type = "fx"; + setter--; + } + + if ( arguments.length < setter ) { + return jQuery.queue( this[0], type ); + } + + return data === undefined ? + this : + this.each(function() { + var queue = jQuery.queue( this, type, data ); + + // ensure a hooks for this queue + jQuery._queueHooks( this, type ); + + if ( type === "fx" && queue[0] !== "inprogress" ) { + jQuery.dequeue( this, type ); + } + }); + }, + dequeue: function( type ) { + return this.each(function() { + jQuery.dequeue( this, type ); + }); + }, + clearQueue: function( type ) { + return this.queue( type || "fx", [] ); + }, + // Get a promise resolved when queues of a certain type + // are emptied (fx is the type by default) + promise: function( type, obj ) { + var tmp, + count = 1, + defer = jQuery.Deferred(), + elements = this, + i = this.length, + resolve = function() { + if ( !( --count ) ) { + defer.resolveWith( elements, [ elements ] ); + } + }; + + if ( typeof type !== "string" ) { + obj = type; + type = undefined; + } + type = type || "fx"; + + while ( i-- ) { + tmp = jQuery._data( elements[ i ], type + "queueHooks" ); + if ( tmp && tmp.empty ) { + count++; + tmp.empty.add( resolve ); + } + } + resolve(); + return defer.promise( obj ); + } +}); +var pnum = (/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/).source; + +var cssExpand = [ "Top", "Right", "Bottom", "Left" ]; + +var isHidden = function( elem, el ) { + // isHidden might be called from jQuery#filter function; + // in that case, element will be second argument + elem = el || elem; + return jQuery.css( elem, "display" ) === "none" || !jQuery.contains( elem.ownerDocument, elem ); + }; + + + +// Multifunctional method to get and set values of a collection +// The value/s can optionally be executed if it's a function +var access = jQuery.access = function( elems, fn, key, value, chainable, emptyGet, raw ) { + var i = 0, + length = elems.length, + bulk = key == null; + + // Sets many values + if ( jQuery.type( key ) === "object" ) { + chainable = true; + for ( i in key ) { + jQuery.access( elems, fn, i, key[i], true, emptyGet, raw ); + } + + // Sets one value + } else if ( value !== undefined ) { + chainable = true; + + if ( !jQuery.isFunction( value ) ) { + raw = true; + } + + if ( bulk ) { + // Bulk operations run against the entire set + if ( raw ) { + fn.call( elems, value ); + fn = null; + + // ...except when executing function values + } else { + bulk = fn; + fn = function( elem, key, value ) { + return bulk.call( jQuery( elem ), value ); + }; + } + } + + if ( fn ) { + for ( ; i < length; i++ ) { + fn( elems[i], key, raw ? value : value.call( elems[i], i, fn( elems[i], key ) ) ); + } + } + } + + return chainable ? + elems : + + // Gets + bulk ? + fn.call( elems ) : + length ? fn( elems[0], key ) : emptyGet; +}; +var rcheckableType = (/^(?:checkbox|radio)$/i); + + + +(function() { + var fragment = document.createDocumentFragment(), + div = document.createElement("div"), + input = document.createElement("input"); + + // Setup + div.setAttribute( "className", "t" ); + div.innerHTML = "
    a"; + + // IE strips leading whitespace when .innerHTML is used + support.leadingWhitespace = div.firstChild.nodeType === 3; + + // Make sure that tbody elements aren't automatically inserted + // IE will insert them into empty tables + support.tbody = !div.getElementsByTagName( "tbody" ).length; + + // Make sure that link elements get serialized correctly by innerHTML + // This requires a wrapper element in IE + support.htmlSerialize = !!div.getElementsByTagName( "link" ).length; + + // Makes sure cloning an html5 element does not cause problems + // Where outerHTML is undefined, this still works + support.html5Clone = + document.createElement( "nav" ).cloneNode( true ).outerHTML !== "<:nav>"; + + // Check if a disconnected checkbox will retain its checked + // value of true after appended to the DOM (IE6/7) + input.type = "checkbox"; + input.checked = true; + fragment.appendChild( input ); + support.appendChecked = input.checked; + + // Make sure textarea (and checkbox) defaultValue is properly cloned + // Support: IE6-IE11+ + div.innerHTML = ""; + support.noCloneChecked = !!div.cloneNode( true ).lastChild.defaultValue; + + // #11217 - WebKit loses check when the name is after the checked attribute + fragment.appendChild( div ); + div.innerHTML = ""; + + // Support: Safari 5.1, iOS 5.1, Android 4.x, Android 2.3 + // old WebKit doesn't clone checked state correctly in fragments + support.checkClone = div.cloneNode( true ).cloneNode( true ).lastChild.checked; + + // Support: IE<9 + // Opera does not clone events (and typeof div.attachEvent === undefined). + // IE9-10 clones events bound via attachEvent, but they don't trigger with .click() + support.noCloneEvent = true; + if ( div.attachEvent ) { + div.attachEvent( "onclick", function() { + support.noCloneEvent = false; + }); + + div.cloneNode( true ).click(); + } + + // Execute the test only if not already executed in another module. + if (support.deleteExpando == null) { + // Support: IE<9 + support.deleteExpando = true; + try { + delete div.test; + } catch( e ) { + support.deleteExpando = false; + } + } + + // Null elements to avoid leaks in IE. + fragment = div = input = null; +})(); + + +(function() { + var i, eventName, + div = document.createElement( "div" ); + + // Support: IE<9 (lack submit/change bubble), Firefox 23+ (lack focusin event) + for ( i in { submit: true, change: true, focusin: true }) { + eventName = "on" + i; + + if ( !(support[ i + "Bubbles" ] = eventName in window) ) { + // Beware of CSP restrictions (https://developer.mozilla.org/en/Security/CSP) + div.setAttribute( eventName, "t" ); + support[ i + "Bubbles" ] = div.attributes[ eventName ].expando === false; + } + } + + // Null elements to avoid leaks in IE. + div = null; +})(); + + +var rformElems = /^(?:input|select|textarea)$/i, + rkeyEvent = /^key/, + rmouseEvent = /^(?:mouse|contextmenu)|click/, + rfocusMorph = /^(?:focusinfocus|focusoutblur)$/, + rtypenamespace = /^([^.]*)(?:\.(.+)|)$/; + +function returnTrue() { + return true; +} + +function returnFalse() { + return false; +} + +function safeActiveElement() { + try { + return document.activeElement; + } catch ( err ) { } +} + +/* + * Helper functions for managing events -- not part of the public interface. + * Props to Dean Edwards' addEvent library for many of the ideas. + */ +jQuery.event = { + + global: {}, + + add: function( elem, types, handler, data, selector ) { + var tmp, events, t, handleObjIn, + special, eventHandle, handleObj, + handlers, type, namespaces, origType, + elemData = jQuery._data( elem ); + + // Don't attach events to noData or text/comment nodes (but allow plain objects) + if ( !elemData ) { + return; + } + + // Caller can pass in an object of custom data in lieu of the handler + if ( handler.handler ) { + handleObjIn = handler; + handler = handleObjIn.handler; + selector = handleObjIn.selector; + } + + // Make sure that the handler has a unique ID, used to find/remove it later + if ( !handler.guid ) { + handler.guid = jQuery.guid++; + } + + // Init the element's event structure and main handler, if this is the first + if ( !(events = elemData.events) ) { + events = elemData.events = {}; + } + if ( !(eventHandle = elemData.handle) ) { + eventHandle = elemData.handle = function( e ) { + // Discard the second event of a jQuery.event.trigger() and + // when an event is called after a page has unloaded + return typeof jQuery !== strundefined && (!e || jQuery.event.triggered !== e.type) ? + jQuery.event.dispatch.apply( eventHandle.elem, arguments ) : + undefined; + }; + // Add elem as a property of the handle fn to prevent a memory leak with IE non-native events + eventHandle.elem = elem; + } + + // Handle multiple events separated by a space + types = ( types || "" ).match( rnotwhite ) || [ "" ]; + t = types.length; + while ( t-- ) { + tmp = rtypenamespace.exec( types[t] ) || []; + type = origType = tmp[1]; + namespaces = ( tmp[2] || "" ).split( "." ).sort(); + + // There *must* be a type, no attaching namespace-only handlers + if ( !type ) { + continue; + } + + // If event changes its type, use the special event handlers for the changed type + special = jQuery.event.special[ type ] || {}; + + // If selector defined, determine special event api type, otherwise given type + type = ( selector ? special.delegateType : special.bindType ) || type; + + // Update special based on newly reset type + special = jQuery.event.special[ type ] || {}; + + // handleObj is passed to all event handlers + handleObj = jQuery.extend({ + type: type, + origType: origType, + data: data, + handler: handler, + guid: handler.guid, + selector: selector, + needsContext: selector && jQuery.expr.match.needsContext.test( selector ), + namespace: namespaces.join(".") + }, handleObjIn ); + + // Init the event handler queue if we're the first + if ( !(handlers = events[ type ]) ) { + handlers = events[ type ] = []; + handlers.delegateCount = 0; + + // Only use addEventListener/attachEvent if the special events handler returns false + if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) { + // Bind the global event handler to the element + if ( elem.addEventListener ) { + elem.addEventListener( type, eventHandle, false ); + + } else if ( elem.attachEvent ) { + elem.attachEvent( "on" + type, eventHandle ); + } + } + } + + if ( special.add ) { + special.add.call( elem, handleObj ); + + if ( !handleObj.handler.guid ) { + handleObj.handler.guid = handler.guid; + } + } + + // Add to the element's handler list, delegates in front + if ( selector ) { + handlers.splice( handlers.delegateCount++, 0, handleObj ); + } else { + handlers.push( handleObj ); + } + + // Keep track of which events have ever been used, for event optimization + jQuery.event.global[ type ] = true; + } + + // Nullify elem to prevent memory leaks in IE + elem = null; + }, + + // Detach an event or set of events from an element + remove: function( elem, types, handler, selector, mappedTypes ) { + var j, handleObj, tmp, + origCount, t, events, + special, handlers, type, + namespaces, origType, + elemData = jQuery.hasData( elem ) && jQuery._data( elem ); + + if ( !elemData || !(events = elemData.events) ) { + return; + } + + // Once for each type.namespace in types; type may be omitted + types = ( types || "" ).match( rnotwhite ) || [ "" ]; + t = types.length; + while ( t-- ) { + tmp = rtypenamespace.exec( types[t] ) || []; + type = origType = tmp[1]; + namespaces = ( tmp[2] || "" ).split( "." ).sort(); + + // Unbind all events (on this namespace, if provided) for the element + if ( !type ) { + for ( type in events ) { + jQuery.event.remove( elem, type + types[ t ], handler, selector, true ); + } + continue; + } + + special = jQuery.event.special[ type ] || {}; + type = ( selector ? special.delegateType : special.bindType ) || type; + handlers = events[ type ] || []; + tmp = tmp[2] && new RegExp( "(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)" ); + + // Remove matching events + origCount = j = handlers.length; + while ( j-- ) { + handleObj = handlers[ j ]; + + if ( ( mappedTypes || origType === handleObj.origType ) && + ( !handler || handler.guid === handleObj.guid ) && + ( !tmp || tmp.test( handleObj.namespace ) ) && + ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) { + handlers.splice( j, 1 ); + + if ( handleObj.selector ) { + handlers.delegateCount--; + } + if ( special.remove ) { + special.remove.call( elem, handleObj ); + } + } + } + + // Remove generic event handler if we removed something and no more handlers exist + // (avoids potential for endless recursion during removal of special event handlers) + if ( origCount && !handlers.length ) { + if ( !special.teardown || special.teardown.call( elem, namespaces, elemData.handle ) === false ) { + jQuery.removeEvent( elem, type, elemData.handle ); + } + + delete events[ type ]; + } + } + + // Remove the expando if it's no longer used + if ( jQuery.isEmptyObject( events ) ) { + delete elemData.handle; + + // removeData also checks for emptiness and clears the expando if empty + // so use it instead of delete + jQuery._removeData( elem, "events" ); + } + }, + + trigger: function( event, data, elem, onlyHandlers ) { + var handle, ontype, cur, + bubbleType, special, tmp, i, + eventPath = [ elem || document ], + type = hasOwn.call( event, "type" ) ? event.type : event, + namespaces = hasOwn.call( event, "namespace" ) ? event.namespace.split(".") : []; + + cur = tmp = elem = elem || document; + + // Don't do events on text and comment nodes + if ( elem.nodeType === 3 || elem.nodeType === 8 ) { + return; + } + + // focus/blur morphs to focusin/out; ensure we're not firing them right now + if ( rfocusMorph.test( type + jQuery.event.triggered ) ) { + return; + } + + if ( type.indexOf(".") >= 0 ) { + // Namespaced trigger; create a regexp to match event type in handle() + namespaces = type.split("."); + type = namespaces.shift(); + namespaces.sort(); + } + ontype = type.indexOf(":") < 0 && "on" + type; + + // Caller can pass in a jQuery.Event object, Object, or just an event type string + event = event[ jQuery.expando ] ? + event : + new jQuery.Event( type, typeof event === "object" && event ); + + // Trigger bitmask: & 1 for native handlers; & 2 for jQuery (always true) + event.isTrigger = onlyHandlers ? 2 : 3; + event.namespace = namespaces.join("."); + event.namespace_re = event.namespace ? + new RegExp( "(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)" ) : + null; + + // Clean up the event in case it is being reused + event.result = undefined; + if ( !event.target ) { + event.target = elem; + } + + // Clone any incoming data and prepend the event, creating the handler arg list + data = data == null ? + [ event ] : + jQuery.makeArray( data, [ event ] ); + + // Allow special events to draw outside the lines + special = jQuery.event.special[ type ] || {}; + if ( !onlyHandlers && special.trigger && special.trigger.apply( elem, data ) === false ) { + return; + } + + // Determine event propagation path in advance, per W3C events spec (#9951) + // Bubble up to document, then to window; watch for a global ownerDocument var (#9724) + if ( !onlyHandlers && !special.noBubble && !jQuery.isWindow( elem ) ) { + + bubbleType = special.delegateType || type; + if ( !rfocusMorph.test( bubbleType + type ) ) { + cur = cur.parentNode; + } + for ( ; cur; cur = cur.parentNode ) { + eventPath.push( cur ); + tmp = cur; + } + + // Only add window if we got to document (e.g., not plain obj or detached DOM) + if ( tmp === (elem.ownerDocument || document) ) { + eventPath.push( tmp.defaultView || tmp.parentWindow || window ); + } + } + + // Fire handlers on the event path + i = 0; + while ( (cur = eventPath[i++]) && !event.isPropagationStopped() ) { + + event.type = i > 1 ? + bubbleType : + special.bindType || type; + + // jQuery handler + handle = ( jQuery._data( cur, "events" ) || {} )[ event.type ] && jQuery._data( cur, "handle" ); + if ( handle ) { + handle.apply( cur, data ); + } + + // Native handler + handle = ontype && cur[ ontype ]; + if ( handle && handle.apply && jQuery.acceptData( cur ) ) { + event.result = handle.apply( cur, data ); + if ( event.result === false ) { + event.preventDefault(); + } + } + } + event.type = type; + + // If nobody prevented the default action, do it now + if ( !onlyHandlers && !event.isDefaultPrevented() ) { + + if ( (!special._default || special._default.apply( eventPath.pop(), data ) === false) && + jQuery.acceptData( elem ) ) { + + // Call a native DOM method on the target with the same name name as the event. + // Can't use an .isFunction() check here because IE6/7 fails that test. + // Don't do default actions on window, that's where global variables be (#6170) + if ( ontype && elem[ type ] && !jQuery.isWindow( elem ) ) { + + // Don't re-trigger an onFOO event when we call its FOO() method + tmp = elem[ ontype ]; + + if ( tmp ) { + elem[ ontype ] = null; + } + + // Prevent re-triggering of the same event, since we already bubbled it above + jQuery.event.triggered = type; + try { + elem[ type ](); + } catch ( e ) { + // IE<9 dies on focus/blur to hidden element (#1486,#12518) + // only reproducible on winXP IE8 native, not IE9 in IE8 mode + } + jQuery.event.triggered = undefined; + + if ( tmp ) { + elem[ ontype ] = tmp; + } + } + } + } + + return event.result; + }, + + dispatch: function( event ) { + + // Make a writable jQuery.Event from the native event object + event = jQuery.event.fix( event ); + + var i, ret, handleObj, matched, j, + handlerQueue = [], + args = slice.call( arguments ), + handlers = ( jQuery._data( this, "events" ) || {} )[ event.type ] || [], + special = jQuery.event.special[ event.type ] || {}; + + // Use the fix-ed jQuery.Event rather than the (read-only) native event + args[0] = event; + event.delegateTarget = this; + + // Call the preDispatch hook for the mapped type, and let it bail if desired + if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) { + return; + } + + // Determine handlers + handlerQueue = jQuery.event.handlers.call( this, event, handlers ); + + // Run delegates first; they may want to stop propagation beneath us + i = 0; + while ( (matched = handlerQueue[ i++ ]) && !event.isPropagationStopped() ) { + event.currentTarget = matched.elem; + + j = 0; + while ( (handleObj = matched.handlers[ j++ ]) && !event.isImmediatePropagationStopped() ) { + + // Triggered event must either 1) have no namespace, or + // 2) have namespace(s) a subset or equal to those in the bound event (both can have no namespace). + if ( !event.namespace_re || event.namespace_re.test( handleObj.namespace ) ) { + + event.handleObj = handleObj; + event.data = handleObj.data; + + ret = ( (jQuery.event.special[ handleObj.origType ] || {}).handle || handleObj.handler ) + .apply( matched.elem, args ); + + if ( ret !== undefined ) { + if ( (event.result = ret) === false ) { + event.preventDefault(); + event.stopPropagation(); + } + } + } + } + } + + // Call the postDispatch hook for the mapped type + if ( special.postDispatch ) { + special.postDispatch.call( this, event ); + } + + return event.result; + }, + + handlers: function( event, handlers ) { + var sel, handleObj, matches, i, + handlerQueue = [], + delegateCount = handlers.delegateCount, + cur = event.target; + + // Find delegate handlers + // Black-hole SVG instance trees (#13180) + // Avoid non-left-click bubbling in Firefox (#3861) + if ( delegateCount && cur.nodeType && (!event.button || event.type !== "click") ) { + + /* jshint eqeqeq: false */ + for ( ; cur != this; cur = cur.parentNode || this ) { + /* jshint eqeqeq: true */ + + // Don't check non-elements (#13208) + // Don't process clicks on disabled elements (#6911, #8165, #11382, #11764) + if ( cur.nodeType === 1 && (cur.disabled !== true || event.type !== "click") ) { + matches = []; + for ( i = 0; i < delegateCount; i++ ) { + handleObj = handlers[ i ]; + + // Don't conflict with Object.prototype properties (#13203) + sel = handleObj.selector + " "; + + if ( matches[ sel ] === undefined ) { + matches[ sel ] = handleObj.needsContext ? + jQuery( sel, this ).index( cur ) >= 0 : + jQuery.find( sel, this, null, [ cur ] ).length; + } + if ( matches[ sel ] ) { + matches.push( handleObj ); + } + } + if ( matches.length ) { + handlerQueue.push({ elem: cur, handlers: matches }); + } + } + } + } + + // Add the remaining (directly-bound) handlers + if ( delegateCount < handlers.length ) { + handlerQueue.push({ elem: this, handlers: handlers.slice( delegateCount ) }); + } + + return handlerQueue; + }, + + fix: function( event ) { + if ( event[ jQuery.expando ] ) { + return event; + } + + // Create a writable copy of the event object and normalize some properties + var i, prop, copy, + type = event.type, + originalEvent = event, + fixHook = this.fixHooks[ type ]; + + if ( !fixHook ) { + this.fixHooks[ type ] = fixHook = + rmouseEvent.test( type ) ? this.mouseHooks : + rkeyEvent.test( type ) ? this.keyHooks : + {}; + } + copy = fixHook.props ? this.props.concat( fixHook.props ) : this.props; + + event = new jQuery.Event( originalEvent ); + + i = copy.length; + while ( i-- ) { + prop = copy[ i ]; + event[ prop ] = originalEvent[ prop ]; + } + + // Support: IE<9 + // Fix target property (#1925) + if ( !event.target ) { + event.target = originalEvent.srcElement || document; + } + + // Support: Chrome 23+, Safari? + // Target should not be a text node (#504, #13143) + if ( event.target.nodeType === 3 ) { + event.target = event.target.parentNode; + } + + // Support: IE<9 + // For mouse/key events, metaKey==false if it's undefined (#3368, #11328) + event.metaKey = !!event.metaKey; + + return fixHook.filter ? fixHook.filter( event, originalEvent ) : event; + }, + + // Includes some event props shared by KeyEvent and MouseEvent + props: "altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "), + + fixHooks: {}, + + keyHooks: { + props: "char charCode key keyCode".split(" "), + filter: function( event, original ) { + + // Add which for key events + if ( event.which == null ) { + event.which = original.charCode != null ? original.charCode : original.keyCode; + } + + return event; + } + }, + + mouseHooks: { + props: "button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "), + filter: function( event, original ) { + var body, eventDoc, doc, + button = original.button, + fromElement = original.fromElement; + + // Calculate pageX/Y if missing and clientX/Y available + if ( event.pageX == null && original.clientX != null ) { + eventDoc = event.target.ownerDocument || document; + doc = eventDoc.documentElement; + body = eventDoc.body; + + event.pageX = original.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 ); + event.pageY = original.clientY + ( doc && doc.scrollTop || body && body.scrollTop || 0 ) - ( doc && doc.clientTop || body && body.clientTop || 0 ); + } + + // Add relatedTarget, if necessary + if ( !event.relatedTarget && fromElement ) { + event.relatedTarget = fromElement === event.target ? original.toElement : fromElement; + } + + // Add which for click: 1 === left; 2 === middle; 3 === right + // Note: button is not normalized, so don't use it + if ( !event.which && button !== undefined ) { + event.which = ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) ); + } + + return event; + } + }, + + special: { + load: { + // Prevent triggered image.load events from bubbling to window.load + noBubble: true + }, + focus: { + // Fire native event if possible so blur/focus sequence is correct + trigger: function() { + if ( this !== safeActiveElement() && this.focus ) { + try { + this.focus(); + return false; + } catch ( e ) { + // Support: IE<9 + // If we error on focus to hidden element (#1486, #12518), + // let .trigger() run the handlers + } + } + }, + delegateType: "focusin" + }, + blur: { + trigger: function() { + if ( this === safeActiveElement() && this.blur ) { + this.blur(); + return false; + } + }, + delegateType: "focusout" + }, + click: { + // For checkbox, fire native event so checked state will be right + trigger: function() { + if ( jQuery.nodeName( this, "input" ) && this.type === "checkbox" && this.click ) { + this.click(); + return false; + } + }, + + // For cross-browser consistency, don't fire native .click() on links + _default: function( event ) { + return jQuery.nodeName( event.target, "a" ); + } + }, + + beforeunload: { + postDispatch: function( event ) { + + // Even when returnValue equals to undefined Firefox will still show alert + if ( event.result !== undefined ) { + event.originalEvent.returnValue = event.result; + } + } + } + }, + + simulate: function( type, elem, event, bubble ) { + // Piggyback on a donor event to simulate a different one. + // Fake originalEvent to avoid donor's stopPropagation, but if the + // simulated event prevents default then we do the same on the donor. + var e = jQuery.extend( + new jQuery.Event(), + event, + { + type: type, + isSimulated: true, + originalEvent: {} + } + ); + if ( bubble ) { + jQuery.event.trigger( e, null, elem ); + } else { + jQuery.event.dispatch.call( elem, e ); + } + if ( e.isDefaultPrevented() ) { + event.preventDefault(); + } + } +}; + +jQuery.removeEvent = document.removeEventListener ? + function( elem, type, handle ) { + if ( elem.removeEventListener ) { + elem.removeEventListener( type, handle, false ); + } + } : + function( elem, type, handle ) { + var name = "on" + type; + + if ( elem.detachEvent ) { + + // #8545, #7054, preventing memory leaks for custom events in IE6-8 + // detachEvent needed property on element, by name of that event, to properly expose it to GC + if ( typeof elem[ name ] === strundefined ) { + elem[ name ] = null; + } + + elem.detachEvent( name, handle ); + } + }; + +jQuery.Event = function( src, props ) { + // Allow instantiation without the 'new' keyword + if ( !(this instanceof jQuery.Event) ) { + return new jQuery.Event( src, props ); + } + + // Event object + if ( src && src.type ) { + this.originalEvent = src; + this.type = src.type; + + // Events bubbling up the document may have been marked as prevented + // by a handler lower down the tree; reflect the correct value. + this.isDefaultPrevented = src.defaultPrevented || + src.defaultPrevented === undefined && ( + // Support: IE < 9 + src.returnValue === false || + // Support: Android < 4.0 + src.getPreventDefault && src.getPreventDefault() ) ? + returnTrue : + returnFalse; + + // Event type + } else { + this.type = src; + } + + // Put explicitly provided properties onto the event object + if ( props ) { + jQuery.extend( this, props ); + } + + // Create a timestamp if incoming event doesn't have one + this.timeStamp = src && src.timeStamp || jQuery.now(); + + // Mark it as fixed + this[ jQuery.expando ] = true; +}; + +// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding +// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html +jQuery.Event.prototype = { + isDefaultPrevented: returnFalse, + isPropagationStopped: returnFalse, + isImmediatePropagationStopped: returnFalse, + + preventDefault: function() { + var e = this.originalEvent; + + this.isDefaultPrevented = returnTrue; + if ( !e ) { + return; + } + + // If preventDefault exists, run it on the original event + if ( e.preventDefault ) { + e.preventDefault(); + + // Support: IE + // Otherwise set the returnValue property of the original event to false + } else { + e.returnValue = false; + } + }, + stopPropagation: function() { + var e = this.originalEvent; + + this.isPropagationStopped = returnTrue; + if ( !e ) { + return; + } + // If stopPropagation exists, run it on the original event + if ( e.stopPropagation ) { + e.stopPropagation(); + } + + // Support: IE + // Set the cancelBubble property of the original event to true + e.cancelBubble = true; + }, + stopImmediatePropagation: function() { + this.isImmediatePropagationStopped = returnTrue; + this.stopPropagation(); + } +}; + +// Create mouseenter/leave events using mouseover/out and event-time checks +jQuery.each({ + mouseenter: "mouseover", + mouseleave: "mouseout" +}, function( orig, fix ) { + jQuery.event.special[ orig ] = { + delegateType: fix, + bindType: fix, + + handle: function( event ) { + var ret, + target = this, + related = event.relatedTarget, + handleObj = event.handleObj; + + // For mousenter/leave call the handler if related is outside the target. + // NB: No relatedTarget if the mouse left/entered the browser window + if ( !related || (related !== target && !jQuery.contains( target, related )) ) { + event.type = handleObj.origType; + ret = handleObj.handler.apply( this, arguments ); + event.type = fix; + } + return ret; + } + }; +}); + +// IE submit delegation +if ( !support.submitBubbles ) { + + jQuery.event.special.submit = { + setup: function() { + // Only need this for delegated form submit events + if ( jQuery.nodeName( this, "form" ) ) { + return false; + } + + // Lazy-add a submit handler when a descendant form may potentially be submitted + jQuery.event.add( this, "click._submit keypress._submit", function( e ) { + // Node name check avoids a VML-related crash in IE (#9807) + var elem = e.target, + form = jQuery.nodeName( elem, "input" ) || jQuery.nodeName( elem, "button" ) ? elem.form : undefined; + if ( form && !jQuery._data( form, "submitBubbles" ) ) { + jQuery.event.add( form, "submit._submit", function( event ) { + event._submit_bubble = true; + }); + jQuery._data( form, "submitBubbles", true ); + } + }); + // return undefined since we don't need an event listener + }, + + postDispatch: function( event ) { + // If form was submitted by the user, bubble the event up the tree + if ( event._submit_bubble ) { + delete event._submit_bubble; + if ( this.parentNode && !event.isTrigger ) { + jQuery.event.simulate( "submit", this.parentNode, event, true ); + } + } + }, + + teardown: function() { + // Only need this for delegated form submit events + if ( jQuery.nodeName( this, "form" ) ) { + return false; + } + + // Remove delegated handlers; cleanData eventually reaps submit handlers attached above + jQuery.event.remove( this, "._submit" ); + } + }; +} + +// IE change delegation and checkbox/radio fix +if ( !support.changeBubbles ) { + + jQuery.event.special.change = { + + setup: function() { + + if ( rformElems.test( this.nodeName ) ) { + // IE doesn't fire change on a check/radio until blur; trigger it on click + // after a propertychange. Eat the blur-change in special.change.handle. + // This still fires onchange a second time for check/radio after blur. + if ( this.type === "checkbox" || this.type === "radio" ) { + jQuery.event.add( this, "propertychange._change", function( event ) { + if ( event.originalEvent.propertyName === "checked" ) { + this._just_changed = true; + } + }); + jQuery.event.add( this, "click._change", function( event ) { + if ( this._just_changed && !event.isTrigger ) { + this._just_changed = false; + } + // Allow triggered, simulated change events (#11500) + jQuery.event.simulate( "change", this, event, true ); + }); + } + return false; + } + // Delegated event; lazy-add a change handler on descendant inputs + jQuery.event.add( this, "beforeactivate._change", function( e ) { + var elem = e.target; + + if ( rformElems.test( elem.nodeName ) && !jQuery._data( elem, "changeBubbles" ) ) { + jQuery.event.add( elem, "change._change", function( event ) { + if ( this.parentNode && !event.isSimulated && !event.isTrigger ) { + jQuery.event.simulate( "change", this.parentNode, event, true ); + } + }); + jQuery._data( elem, "changeBubbles", true ); + } + }); + }, + + handle: function( event ) { + var elem = event.target; + + // Swallow native change events from checkbox/radio, we already triggered them above + if ( this !== elem || event.isSimulated || event.isTrigger || (elem.type !== "radio" && elem.type !== "checkbox") ) { + return event.handleObj.handler.apply( this, arguments ); + } + }, + + teardown: function() { + jQuery.event.remove( this, "._change" ); + + return !rformElems.test( this.nodeName ); + } + }; +} + +// Create "bubbling" focus and blur events +if ( !support.focusinBubbles ) { + jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) { + + // Attach a single capturing handler on the document while someone wants focusin/focusout + var handler = function( event ) { + jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ), true ); + }; + + jQuery.event.special[ fix ] = { + setup: function() { + var doc = this.ownerDocument || this, + attaches = jQuery._data( doc, fix ); + + if ( !attaches ) { + doc.addEventListener( orig, handler, true ); + } + jQuery._data( doc, fix, ( attaches || 0 ) + 1 ); + }, + teardown: function() { + var doc = this.ownerDocument || this, + attaches = jQuery._data( doc, fix ) - 1; + + if ( !attaches ) { + doc.removeEventListener( orig, handler, true ); + jQuery._removeData( doc, fix ); + } else { + jQuery._data( doc, fix, attaches ); + } + } + }; + }); +} + +jQuery.fn.extend({ + + on: function( types, selector, data, fn, /*INTERNAL*/ one ) { + var type, origFn; + + // Types can be a map of types/handlers + if ( typeof types === "object" ) { + // ( types-Object, selector, data ) + if ( typeof selector !== "string" ) { + // ( types-Object, data ) + data = data || selector; + selector = undefined; + } + for ( type in types ) { + this.on( type, selector, data, types[ type ], one ); + } + return this; + } + + if ( data == null && fn == null ) { + // ( types, fn ) + fn = selector; + data = selector = undefined; + } else if ( fn == null ) { + if ( typeof selector === "string" ) { + // ( types, selector, fn ) + fn = data; + data = undefined; + } else { + // ( types, data, fn ) + fn = data; + data = selector; + selector = undefined; + } + } + if ( fn === false ) { + fn = returnFalse; + } else if ( !fn ) { + return this; + } + + if ( one === 1 ) { + origFn = fn; + fn = function( event ) { + // Can use an empty set, since event contains the info + jQuery().off( event ); + return origFn.apply( this, arguments ); + }; + // Use same guid so caller can remove using origFn + fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ ); + } + return this.each( function() { + jQuery.event.add( this, types, fn, data, selector ); + }); + }, + one: function( types, selector, data, fn ) { + return this.on( types, selector, data, fn, 1 ); + }, + off: function( types, selector, fn ) { + var handleObj, type; + if ( types && types.preventDefault && types.handleObj ) { + // ( event ) dispatched jQuery.Event + handleObj = types.handleObj; + jQuery( types.delegateTarget ).off( + handleObj.namespace ? handleObj.origType + "." + handleObj.namespace : handleObj.origType, + handleObj.selector, + handleObj.handler + ); + return this; + } + if ( typeof types === "object" ) { + // ( types-object [, selector] ) + for ( type in types ) { + this.off( type, selector, types[ type ] ); + } + return this; + } + if ( selector === false || typeof selector === "function" ) { + // ( types [, fn] ) + fn = selector; + selector = undefined; + } + if ( fn === false ) { + fn = returnFalse; + } + return this.each(function() { + jQuery.event.remove( this, types, fn, selector ); + }); + }, + + trigger: function( type, data ) { + return this.each(function() { + jQuery.event.trigger( type, data, this ); + }); + }, + triggerHandler: function( type, data ) { + var elem = this[0]; + if ( elem ) { + return jQuery.event.trigger( type, data, elem, true ); + } + } +}); + + +function createSafeFragment( document ) { + var list = nodeNames.split( "|" ), + safeFrag = document.createDocumentFragment(); + + if ( safeFrag.createElement ) { + while ( list.length ) { + safeFrag.createElement( + list.pop() + ); + } + } + return safeFrag; +} + +var nodeNames = "abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|" + + "header|hgroup|mark|meter|nav|output|progress|section|summary|time|video", + rinlinejQuery = / jQuery\d+="(?:null|\d+)"/g, + rnoshimcache = new RegExp("<(?:" + nodeNames + ")[\\s/>]", "i"), + rleadingWhitespace = /^\s+/, + rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi, + rtagName = /<([\w:]+)/, + rtbody = /\s*$/g, + + // We have to close these tags to support XHTML (#13200) + wrapMap = { + option: [ 1, "" ], + legend: [ 1, "
    ", "
    " ], + area: [ 1, "", "" ], + param: [ 1, "", "" ], + thead: [ 1, "", "
    " ], + tr: [ 2, "", "
    " ], + col: [ 2, "", "
    " ], + td: [ 3, "", "
    " ], + + // IE6-8 can't serialize link, script, style, or any html5 (NoScope) tags, + // unless wrapped in a div with non-breaking characters in front of it. + _default: support.htmlSerialize ? [ 0, "", "" ] : [ 1, "X
    ", "
    " ] + }, + safeFragment = createSafeFragment( document ), + fragmentDiv = safeFragment.appendChild( document.createElement("div") ); + +wrapMap.optgroup = wrapMap.option; +wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; +wrapMap.th = wrapMap.td; + +function getAll( context, tag ) { + var elems, elem, + i = 0, + found = typeof context.getElementsByTagName !== strundefined ? context.getElementsByTagName( tag || "*" ) : + typeof context.querySelectorAll !== strundefined ? context.querySelectorAll( tag || "*" ) : + undefined; + + if ( !found ) { + for ( found = [], elems = context.childNodes || context; (elem = elems[i]) != null; i++ ) { + if ( !tag || jQuery.nodeName( elem, tag ) ) { + found.push( elem ); + } else { + jQuery.merge( found, getAll( elem, tag ) ); + } + } + } + + return tag === undefined || tag && jQuery.nodeName( context, tag ) ? + jQuery.merge( [ context ], found ) : + found; +} + +// Used in buildFragment, fixes the defaultChecked property +function fixDefaultChecked( elem ) { + if ( rcheckableType.test( elem.type ) ) { + elem.defaultChecked = elem.checked; + } +} + +// Support: IE<8 +// Manipulating tables requires a tbody +function manipulationTarget( elem, content ) { + return jQuery.nodeName( elem, "table" ) && + jQuery.nodeName( content.nodeType !== 11 ? content : content.firstChild, "tr" ) ? + + elem.getElementsByTagName("tbody")[0] || + elem.appendChild( elem.ownerDocument.createElement("tbody") ) : + elem; +} + +// Replace/restore the type attribute of script elements for safe DOM manipulation +function disableScript( elem ) { + elem.type = (jQuery.find.attr( elem, "type" ) !== null) + "/" + elem.type; + return elem; +} +function restoreScript( elem ) { + var match = rscriptTypeMasked.exec( elem.type ); + if ( match ) { + elem.type = match[1]; + } else { + elem.removeAttribute("type"); + } + return elem; +} + +// Mark scripts as having already been evaluated +function setGlobalEval( elems, refElements ) { + var elem, + i = 0; + for ( ; (elem = elems[i]) != null; i++ ) { + jQuery._data( elem, "globalEval", !refElements || jQuery._data( refElements[i], "globalEval" ) ); + } +} + +function cloneCopyEvent( src, dest ) { + + if ( dest.nodeType !== 1 || !jQuery.hasData( src ) ) { + return; + } + + var type, i, l, + oldData = jQuery._data( src ), + curData = jQuery._data( dest, oldData ), + events = oldData.events; + + if ( events ) { + delete curData.handle; + curData.events = {}; + + for ( type in events ) { + for ( i = 0, l = events[ type ].length; i < l; i++ ) { + jQuery.event.add( dest, type, events[ type ][ i ] ); + } + } + } + + // make the cloned public data object a copy from the original + if ( curData.data ) { + curData.data = jQuery.extend( {}, curData.data ); + } +} + +function fixCloneNodeIssues( src, dest ) { + var nodeName, e, data; + + // We do not need to do anything for non-Elements + if ( dest.nodeType !== 1 ) { + return; + } + + nodeName = dest.nodeName.toLowerCase(); + + // IE6-8 copies events bound via attachEvent when using cloneNode. + if ( !support.noCloneEvent && dest[ jQuery.expando ] ) { + data = jQuery._data( dest ); + + for ( e in data.events ) { + jQuery.removeEvent( dest, e, data.handle ); + } + + // Event data gets referenced instead of copied if the expando gets copied too + dest.removeAttribute( jQuery.expando ); + } + + // IE blanks contents when cloning scripts, and tries to evaluate newly-set text + if ( nodeName === "script" && dest.text !== src.text ) { + disableScript( dest ).text = src.text; + restoreScript( dest ); + + // IE6-10 improperly clones children of object elements using classid. + // IE10 throws NoModificationAllowedError if parent is null, #12132. + } else if ( nodeName === "object" ) { + if ( dest.parentNode ) { + dest.outerHTML = src.outerHTML; + } + + // This path appears unavoidable for IE9. When cloning an object + // element in IE9, the outerHTML strategy above is not sufficient. + // If the src has innerHTML and the destination does not, + // copy the src.innerHTML into the dest.innerHTML. #10324 + if ( support.html5Clone && ( src.innerHTML && !jQuery.trim(dest.innerHTML) ) ) { + dest.innerHTML = src.innerHTML; + } + + } else if ( nodeName === "input" && rcheckableType.test( src.type ) ) { + // IE6-8 fails to persist the checked state of a cloned checkbox + // or radio button. Worse, IE6-7 fail to give the cloned element + // a checked appearance if the defaultChecked value isn't also set + + dest.defaultChecked = dest.checked = src.checked; + + // IE6-7 get confused and end up setting the value of a cloned + // checkbox/radio button to an empty string instead of "on" + if ( dest.value !== src.value ) { + dest.value = src.value; + } + + // IE6-8 fails to return the selected option to the default selected + // state when cloning options + } else if ( nodeName === "option" ) { + dest.defaultSelected = dest.selected = src.defaultSelected; + + // IE6-8 fails to set the defaultValue to the correct value when + // cloning other types of input fields + } else if ( nodeName === "input" || nodeName === "textarea" ) { + dest.defaultValue = src.defaultValue; + } +} + +jQuery.extend({ + clone: function( elem, dataAndEvents, deepDataAndEvents ) { + var destElements, node, clone, i, srcElements, + inPage = jQuery.contains( elem.ownerDocument, elem ); + + if ( support.html5Clone || jQuery.isXMLDoc(elem) || !rnoshimcache.test( "<" + elem.nodeName + ">" ) ) { + clone = elem.cloneNode( true ); + + // IE<=8 does not properly clone detached, unknown element nodes + } else { + fragmentDiv.innerHTML = elem.outerHTML; + fragmentDiv.removeChild( clone = fragmentDiv.firstChild ); + } + + if ( (!support.noCloneEvent || !support.noCloneChecked) && + (elem.nodeType === 1 || elem.nodeType === 11) && !jQuery.isXMLDoc(elem) ) { + + // We eschew Sizzle here for performance reasons: http://jsperf.com/getall-vs-sizzle/2 + destElements = getAll( clone ); + srcElements = getAll( elem ); + + // Fix all IE cloning issues + for ( i = 0; (node = srcElements[i]) != null; ++i ) { + // Ensure that the destination node is not null; Fixes #9587 + if ( destElements[i] ) { + fixCloneNodeIssues( node, destElements[i] ); + } + } + } + + // Copy the events from the original to the clone + if ( dataAndEvents ) { + if ( deepDataAndEvents ) { + srcElements = srcElements || getAll( elem ); + destElements = destElements || getAll( clone ); + + for ( i = 0; (node = srcElements[i]) != null; i++ ) { + cloneCopyEvent( node, destElements[i] ); + } + } else { + cloneCopyEvent( elem, clone ); + } + } + + // Preserve script evaluation history + destElements = getAll( clone, "script" ); + if ( destElements.length > 0 ) { + setGlobalEval( destElements, !inPage && getAll( elem, "script" ) ); + } + + destElements = srcElements = node = null; + + // Return the cloned set + return clone; + }, + + buildFragment: function( elems, context, scripts, selection ) { + var j, elem, contains, + tmp, tag, tbody, wrap, + l = elems.length, + + // Ensure a safe fragment + safe = createSafeFragment( context ), + + nodes = [], + i = 0; + + for ( ; i < l; i++ ) { + elem = elems[ i ]; + + if ( elem || elem === 0 ) { + + // Add nodes directly + if ( jQuery.type( elem ) === "object" ) { + jQuery.merge( nodes, elem.nodeType ? [ elem ] : elem ); + + // Convert non-html into a text node + } else if ( !rhtml.test( elem ) ) { + nodes.push( context.createTextNode( elem ) ); + + // Convert html into DOM nodes + } else { + tmp = tmp || safe.appendChild( context.createElement("div") ); + + // Deserialize a standard representation + tag = (rtagName.exec( elem ) || [ "", "" ])[ 1 ].toLowerCase(); + wrap = wrapMap[ tag ] || wrapMap._default; + + tmp.innerHTML = wrap[1] + elem.replace( rxhtmlTag, "<$1>" ) + wrap[2]; + + // Descend through wrappers to the right content + j = wrap[0]; + while ( j-- ) { + tmp = tmp.lastChild; + } + + // Manually add leading whitespace removed by IE + if ( !support.leadingWhitespace && rleadingWhitespace.test( elem ) ) { + nodes.push( context.createTextNode( rleadingWhitespace.exec( elem )[0] ) ); + } + + // Remove IE's autoinserted from table fragments + if ( !support.tbody ) { + + // String was a , *may* have spurious + elem = tag === "table" && !rtbody.test( elem ) ? + tmp.firstChild : + + // String was a bare or + wrap[1] === "
    " && !rtbody.test( elem ) ? + tmp : + 0; + + j = elem && elem.childNodes.length; + while ( j-- ) { + if ( jQuery.nodeName( (tbody = elem.childNodes[j]), "tbody" ) && !tbody.childNodes.length ) { + elem.removeChild( tbody ); + } + } + } + + jQuery.merge( nodes, tmp.childNodes ); + + // Fix #12392 for WebKit and IE > 9 + tmp.textContent = ""; + + // Fix #12392 for oldIE + while ( tmp.firstChild ) { + tmp.removeChild( tmp.firstChild ); + } + + // Remember the top-level container for proper cleanup + tmp = safe.lastChild; + } + } + } + + // Fix #11356: Clear elements from fragment + if ( tmp ) { + safe.removeChild( tmp ); + } + + // Reset defaultChecked for any radios and checkboxes + // about to be appended to the DOM in IE 6/7 (#8060) + if ( !support.appendChecked ) { + jQuery.grep( getAll( nodes, "input" ), fixDefaultChecked ); + } + + i = 0; + while ( (elem = nodes[ i++ ]) ) { + + // #4087 - If origin and destination elements are the same, and this is + // that element, do not do anything + if ( selection && jQuery.inArray( elem, selection ) !== -1 ) { + continue; + } + + contains = jQuery.contains( elem.ownerDocument, elem ); + + // Append to fragment + tmp = getAll( safe.appendChild( elem ), "script" ); + + // Preserve script evaluation history + if ( contains ) { + setGlobalEval( tmp ); + } + + // Capture executables + if ( scripts ) { + j = 0; + while ( (elem = tmp[ j++ ]) ) { + if ( rscriptType.test( elem.type || "" ) ) { + scripts.push( elem ); + } + } + } + } + + tmp = null; + + return safe; + }, + + cleanData: function( elems, /* internal */ acceptData ) { + var elem, type, id, data, + i = 0, + internalKey = jQuery.expando, + cache = jQuery.cache, + deleteExpando = support.deleteExpando, + special = jQuery.event.special; + + for ( ; (elem = elems[i]) != null; i++ ) { + if ( acceptData || jQuery.acceptData( elem ) ) { + + id = elem[ internalKey ]; + data = id && cache[ id ]; + + if ( data ) { + if ( data.events ) { + for ( type in data.events ) { + if ( special[ type ] ) { + jQuery.event.remove( elem, type ); + + // This is a shortcut to avoid jQuery.event.remove's overhead + } else { + jQuery.removeEvent( elem, type, data.handle ); + } + } + } + + // Remove cache only if it was not already removed by jQuery.event.remove + if ( cache[ id ] ) { + + delete cache[ id ]; + + // IE does not allow us to delete expando properties from nodes, + // nor does it have a removeAttribute function on Document nodes; + // we must handle all of these cases + if ( deleteExpando ) { + delete elem[ internalKey ]; + + } else if ( typeof elem.removeAttribute !== strundefined ) { + elem.removeAttribute( internalKey ); + + } else { + elem[ internalKey ] = null; + } + + deletedIds.push( id ); + } + } + } + } + } +}); + +jQuery.fn.extend({ + text: function( value ) { + return access( this, function( value ) { + return value === undefined ? + jQuery.text( this ) : + this.empty().append( ( this[0] && this[0].ownerDocument || document ).createTextNode( value ) ); + }, null, value, arguments.length ); + }, + + append: function() { + return this.domManip( arguments, function( elem ) { + if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { + var target = manipulationTarget( this, elem ); + target.appendChild( elem ); + } + }); + }, + + prepend: function() { + return this.domManip( arguments, function( elem ) { + if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { + var target = manipulationTarget( this, elem ); + target.insertBefore( elem, target.firstChild ); + } + }); + }, + + before: function() { + return this.domManip( arguments, function( elem ) { + if ( this.parentNode ) { + this.parentNode.insertBefore( elem, this ); + } + }); + }, + + after: function() { + return this.domManip( arguments, function( elem ) { + if ( this.parentNode ) { + this.parentNode.insertBefore( elem, this.nextSibling ); + } + }); + }, + + remove: function( selector, keepData /* Internal Use Only */ ) { + var elem, + elems = selector ? jQuery.filter( selector, this ) : this, + i = 0; + + for ( ; (elem = elems[i]) != null; i++ ) { + + if ( !keepData && elem.nodeType === 1 ) { + jQuery.cleanData( getAll( elem ) ); + } + + if ( elem.parentNode ) { + if ( keepData && jQuery.contains( elem.ownerDocument, elem ) ) { + setGlobalEval( getAll( elem, "script" ) ); + } + elem.parentNode.removeChild( elem ); + } + } + + return this; + }, + + empty: function() { + var elem, + i = 0; + + for ( ; (elem = this[i]) != null; i++ ) { + // Remove element nodes and prevent memory leaks + if ( elem.nodeType === 1 ) { + jQuery.cleanData( getAll( elem, false ) ); + } + + // Remove any remaining nodes + while ( elem.firstChild ) { + elem.removeChild( elem.firstChild ); + } + + // If this is a select, ensure that it displays empty (#12336) + // Support: IE<9 + if ( elem.options && jQuery.nodeName( elem, "select" ) ) { + elem.options.length = 0; + } + } + + return this; + }, + + clone: function( dataAndEvents, deepDataAndEvents ) { + dataAndEvents = dataAndEvents == null ? false : dataAndEvents; + deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents; + + return this.map(function() { + return jQuery.clone( this, dataAndEvents, deepDataAndEvents ); + }); + }, + + html: function( value ) { + return access( this, function( value ) { + var elem = this[ 0 ] || {}, + i = 0, + l = this.length; + + if ( value === undefined ) { + return elem.nodeType === 1 ? + elem.innerHTML.replace( rinlinejQuery, "" ) : + undefined; + } + + // See if we can take a shortcut and just use innerHTML + if ( typeof value === "string" && !rnoInnerhtml.test( value ) && + ( support.htmlSerialize || !rnoshimcache.test( value ) ) && + ( support.leadingWhitespace || !rleadingWhitespace.test( value ) ) && + !wrapMap[ (rtagName.exec( value ) || [ "", "" ])[ 1 ].toLowerCase() ] ) { + + value = value.replace( rxhtmlTag, "<$1>" ); + + try { + for (; i < l; i++ ) { + // Remove element nodes and prevent memory leaks + elem = this[i] || {}; + if ( elem.nodeType === 1 ) { + jQuery.cleanData( getAll( elem, false ) ); + elem.innerHTML = value; + } + } + + elem = 0; + + // If using innerHTML throws an exception, use the fallback method + } catch(e) {} + } + + if ( elem ) { + this.empty().append( value ); + } + }, null, value, arguments.length ); + }, + + replaceWith: function() { + var arg = arguments[ 0 ]; + + // Make the changes, replacing each context element with the new content + this.domManip( arguments, function( elem ) { + arg = this.parentNode; + + jQuery.cleanData( getAll( this ) ); + + if ( arg ) { + arg.replaceChild( elem, this ); + } + }); + + // Force removal if there was no new content (e.g., from empty arguments) + return arg && (arg.length || arg.nodeType) ? this : this.remove(); + }, + + detach: function( selector ) { + return this.remove( selector, true ); + }, + + domManip: function( args, callback ) { + + // Flatten any nested arrays + args = concat.apply( [], args ); + + var first, node, hasScripts, + scripts, doc, fragment, + i = 0, + l = this.length, + set = this, + iNoClone = l - 1, + value = args[0], + isFunction = jQuery.isFunction( value ); + + // We can't cloneNode fragments that contain checked, in WebKit + if ( isFunction || + ( l > 1 && typeof value === "string" && + !support.checkClone && rchecked.test( value ) ) ) { + return this.each(function( index ) { + var self = set.eq( index ); + if ( isFunction ) { + args[0] = value.call( this, index, self.html() ); + } + self.domManip( args, callback ); + }); + } + + if ( l ) { + fragment = jQuery.buildFragment( args, this[ 0 ].ownerDocument, false, this ); + first = fragment.firstChild; + + if ( fragment.childNodes.length === 1 ) { + fragment = first; + } + + if ( first ) { + scripts = jQuery.map( getAll( fragment, "script" ), disableScript ); + hasScripts = scripts.length; + + // Use the original fragment for the last item instead of the first because it can end up + // being emptied incorrectly in certain situations (#8070). + for ( ; i < l; i++ ) { + node = fragment; + + if ( i !== iNoClone ) { + node = jQuery.clone( node, true, true ); + + // Keep references to cloned scripts for later restoration + if ( hasScripts ) { + jQuery.merge( scripts, getAll( node, "script" ) ); + } + } + + callback.call( this[i], node, i ); + } + + if ( hasScripts ) { + doc = scripts[ scripts.length - 1 ].ownerDocument; + + // Reenable scripts + jQuery.map( scripts, restoreScript ); + + // Evaluate executable scripts on first document insertion + for ( i = 0; i < hasScripts; i++ ) { + node = scripts[ i ]; + if ( rscriptType.test( node.type || "" ) && + !jQuery._data( node, "globalEval" ) && jQuery.contains( doc, node ) ) { + + if ( node.src ) { + // Optional AJAX dependency, but won't run scripts if not present + if ( jQuery._evalUrl ) { + jQuery._evalUrl( node.src ); + } + } else { + jQuery.globalEval( ( node.text || node.textContent || node.innerHTML || "" ).replace( rcleanScript, "" ) ); + } + } + } + } + + // Fix #11809: Avoid leaking memory + fragment = first = null; + } + } + + return this; + } +}); + +jQuery.each({ + appendTo: "append", + prependTo: "prepend", + insertBefore: "before", + insertAfter: "after", + replaceAll: "replaceWith" +}, function( name, original ) { + jQuery.fn[ name ] = function( selector ) { + var elems, + i = 0, + ret = [], + insert = jQuery( selector ), + last = insert.length - 1; + + for ( ; i <= last; i++ ) { + elems = i === last ? this : this.clone(true); + jQuery( insert[i] )[ original ]( elems ); + + // Modern browsers can apply jQuery collections as arrays, but oldIE needs a .get() + push.apply( ret, elems.get() ); + } + + return this.pushStack( ret ); + }; +}); + + +var iframe, + elemdisplay = {}; + +/** + * Retrieve the actual display of a element + * @param {String} name nodeName of the element + * @param {Object} doc Document object + */ +// Called only from within defaultDisplay +function actualDisplay( name, doc ) { + var elem = jQuery( doc.createElement( name ) ).appendTo( doc.body ), + + // getDefaultComputedStyle might be reliably used only on attached element + display = window.getDefaultComputedStyle ? + + // Use of this method is a temporary fix (more like optmization) until something better comes along, + // since it was removed from specification and supported only in FF + window.getDefaultComputedStyle( elem[ 0 ] ).display : jQuery.css( elem[ 0 ], "display" ); + + // We don't have any data stored on the element, + // so use "detach" method as fast way to get rid of the element + elem.detach(); + + return display; +} + +/** + * Try to determine the default display value of an element + * @param {String} nodeName + */ +function defaultDisplay( nodeName ) { + var doc = document, + display = elemdisplay[ nodeName ]; + + if ( !display ) { + display = actualDisplay( nodeName, doc ); + + // If the simple way fails, read from inside an iframe + if ( display === "none" || !display ) { + + // Use the already-created iframe if possible + iframe = (iframe || jQuery( "