local/cloudly-min
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

overdue

Browse Source
This commit is contained in:
bel
2020-01-13 03:37:51 +00:00
commit c8eb52f9ba
2023 changed files with 702080 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

259
.rclone_repo/cmd/serve/http/http.go Executable file
View File

@@ -0,0 +1,259 @@
package http
import (
"fmt"
"html/template"
"net/http"
"os"
"path"
"strconv"
"strings"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/serve/httplib"
"github.com/ncw/rclone/cmd/serve/httplib/httpflags"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/accounting"
"github.com/ncw/rclone/lib/rest"
"github.com/ncw/rclone/vfs"
"github.com/ncw/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
)
func init() {
httpflags.AddFlags(Command.Flags())
vfsflags.AddFlags(Command.Flags())
}
// Command definition for cobra
var Command = &cobra.Command{
Use: "http remote:path",
Short: `Serve the remote over HTTP.`,
Long: `rclone serve http implements a basic web server to serve the remote
over HTTP. This can be viewed in a web browser or you can make a
remote of type http read from it.
You can use the filter flags (eg --include, --exclude) to control what
is served.
The server will log errors. Use -v to see access logs.
--bwlimit will be respected for file transfers. Use --stats to
control the stats printing.
` + httplib.Help + vfs.Help,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
f := cmd.NewFsSrc(args)
cmd.Run(false, true, command, func() error {
s := newServer(f, &httpflags.Opt)
s.serve()
return nil
})
},
}
// server contains everything to run the server
type server struct {
f fs.Fs
vfs *vfs.VFS
srv *httplib.Server
}
func newServer(f fs.Fs, opt *httplib.Options) *server {
mux := http.NewServeMux()
s := &server{
f: f,
vfs: vfs.New(f, &vfsflags.Opt),
srv: httplib.NewServer(mux, opt),
}
mux.HandleFunc("/", s.handler)
return s
}
// serve runs the http server - doesn't return
func (s *server) serve() {
err := s.srv.Serve()
if err != nil {
fs.Errorf(s.f, "Opening listener: %v", err)
}
fs.Logf(s.f, "Serving on %s", s.srv.URL())
s.srv.Wait()
}
// handler reads incoming requests and dispatches them
func (s *server) handler(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" && r.Method != "HEAD" {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
w.Header().Set("Accept-Ranges", "bytes")
w.Header().Set("Server", "rclone/"+fs.Version)
urlPath := r.URL.Path
isDir := strings.HasSuffix(urlPath, "/")
remote := strings.Trim(urlPath, "/")
if isDir {
s.serveDir(w, r, remote)
} else {
s.serveFile(w, r, remote)
}
}
// entry is a directory entry
type entry struct {
remote string
URL string
Leaf string
}
// entries represents a directory
type entries []entry
// addEntry adds an entry to that directory
func (es *entries) addEntry(node interface {
Path() string
Name() string
IsDir() bool
}) {
remote := node.Path()
leaf := node.Name()
urlRemote := leaf
if node.IsDir() {
leaf += "/"
urlRemote += "/"
}
*es = append(*es, entry{remote: remote, URL: rest.URLPathEscape(urlRemote), Leaf: leaf})
}
// indexPage is a directory listing template
var indexPage = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{{ .Title }}</title>
</head>
<body>
<h1>{{ .Title }}</h1>
{{ range $i := .Entries }}<a href="{{ $i.URL }}">{{ $i.Leaf }}</a><br />
{{ end }}</body>
</html>
`
// indexTemplate is the instantiated indexPage
var indexTemplate = template.Must(template.New("index").Parse(indexPage))
// indexData is used to fill in the indexTemplate
type indexData struct {
Title string
Entries entries
}
// error returns an http.StatusInternalServerError and logs the error
func internalError(what interface{}, w http.ResponseWriter, text string, err error) {
fs.CountError(err)
fs.Errorf(what, "%s: %v", text, err)
http.Error(w, text+".", http.StatusInternalServerError)
}
// serveDir serves a directory index at dirRemote
func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
// List the directory
node, err := s.vfs.Stat(dirRemote)
if err == vfs.ENOENT {
http.Error(w, "Directory not found", http.StatusNotFound)
return
} else if err != nil {
internalError(dirRemote, w, "Failed to list directory", err)
return
}
if !node.IsDir() {
http.Error(w, "Not a directory", http.StatusNotFound)
return
}
dir := node.(*vfs.Dir)
dirEntries, err := dir.ReadDirAll()
if err != nil {
internalError(dirRemote, w, "Failed to list directory", err)
return
}
var out entries
for _, node := range dirEntries {
out.addEntry(node)
}
// Account the transfer
accounting.Stats.Transferring(dirRemote)
defer accounting.Stats.DoneTransferring(dirRemote, true)
fs.Infof(dirRemote, "%s: Serving directory", r.RemoteAddr)
err = indexTemplate.Execute(w, indexData{
Entries: out,
Title: fmt.Sprintf("Directory listing of /%s", dirRemote),
})
if err != nil {
internalError(dirRemote, w, "Failed to render template", err)
return
}
}
// serveFile serves a file object at remote
func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string) {
node, err := s.vfs.Stat(remote)
if err == vfs.ENOENT {
fs.Infof(remote, "%s: File not found", r.RemoteAddr)
http.Error(w, "File not found", http.StatusNotFound)
return
} else if err != nil {
internalError(remote, w, "Failed to find file", err)
return
}
if !node.IsFile() {
http.Error(w, "Not a file", http.StatusNotFound)
return
}
entry := node.DirEntry()
if entry == nil {
http.Error(w, "Can't open file being written", http.StatusNotFound)
return
}
obj := entry.(fs.Object)
file := node.(*vfs.File)
// Set content length since we know how long the object is
w.Header().Set("Content-Length", strconv.FormatInt(node.Size(), 10))
// Set content type
mimeType := fs.MimeType(obj)
if mimeType == "application/octet-stream" && path.Ext(remote) == "" {
// Leave header blank so http server guesses
} else {
w.Header().Set("Content-Type", mimeType)
}
// If HEAD no need to read the object since we have set the headers
if r.Method == "HEAD" {
return
}
// open the object
in, err := file.Open(os.O_RDONLY)
if err != nil {
internalError(remote, w, "Failed to open file", err)
return
}
defer func() {
err := in.Close()
if err != nil {
fs.Errorf(remote, "Failed to close file: %v", err)
}
}()
// Account the transfer
accounting.Stats.Transferring(remote)
defer accounting.Stats.DoneTransferring(remote, true)
// FIXME in = fs.NewAccount(in, obj).WithBuffer() // account the transfer
// Serve the file
http.ServeContent(w, r, remote, node.ModTime(), in)
}

237
.rclone_repo/cmd/serve/http/http_test.go Executable file
View File

@@ -0,0 +1,237 @@
// +build go1.8
package http
import (
"flag"
"io/ioutil"
"net"
"net/http"
"path"
"strings"
"testing"
"time"
_ "github.com/ncw/rclone/backend/local"
"github.com/ncw/rclone/cmd/serve/httplib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/config"
"github.com/ncw/rclone/fs/filter"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var (
updateGolden = flag.Bool("updategolden", false, "update golden files for regression test")
httpServer *server
)
const (
testBindAddress = "localhost:51777"
testURL = "http://" + testBindAddress + "/"
)
func startServer(t *testing.T, f fs.Fs) {
opt := httplib.DefaultOpt
opt.ListenAddr = testBindAddress
httpServer = newServer(f, &opt)
go httpServer.serve()
// try to connect to the test server
pause := time.Millisecond
for i := 0; i < 10; i++ {
conn, err := net.Dial("tcp", testBindAddress)
if err == nil {
_ = conn.Close()
return
}
// t.Logf("couldn't connect, sleeping for %v: %v", pause, err)
time.Sleep(pause)
pause *= 2
}
t.Fatal("couldn't connect to server")
}
func TestInit(t *testing.T) {
// Configure the remote
config.LoadConfig()
// fs.Config.LogLevel = fs.LogLevelDebug
// fs.Config.DumpHeaders = true
// fs.Config.DumpBodies = true
// exclude files called hidden.txt and directories called hidden
require.NoError(t, filter.Active.AddRule("- hidden.txt"))
require.NoError(t, filter.Active.AddRule("- hidden/**"))
// Create a test Fs
f, err := fs.NewFs("testdata/files")
require.NoError(t, err)
startServer(t, f)
}
// check body against the file, or re-write body if -updategolden is
// set.
func checkGolden(t *testing.T, fileName string, got []byte) {
if *updateGolden {
t.Logf("Updating golden file %q", fileName)
err := ioutil.WriteFile(fileName, got, 0666)
require.NoError(t, err)
} else {
want, err := ioutil.ReadFile(fileName)
require.NoError(t, err)
wants := strings.Split(string(want), "\n")
gots := strings.Split(string(got), "\n")
assert.Equal(t, wants, gots, fileName)
}
}
func TestGET(t *testing.T) {
for _, test := range []struct {
URL string
Status int
Golden string
Method string
Range string
}{
{
URL: "",
Status: http.StatusOK,
Golden: "testdata/golden/index.html",
},
{
URL: "notfound",
Status: http.StatusNotFound,
Golden: "testdata/golden/notfound.html",
},
{
URL: "dirnotfound/",
Status: http.StatusNotFound,
Golden: "testdata/golden/dirnotfound.html",
},
{
URL: "hidden/",
Status: http.StatusNotFound,
Golden: "testdata/golden/hiddendir.html",
},
{
URL: "one%25.txt",
Status: http.StatusOK,
Golden: "testdata/golden/one.txt",
},
{
URL: "hidden.txt",
Status: http.StatusNotFound,
Golden: "testdata/golden/hidden.txt",
},
{
URL: "three/",
Status: http.StatusOK,
Golden: "testdata/golden/three.html",
},
{
URL: "three/a.txt",
Status: http.StatusOK,
Golden: "testdata/golden/a.txt",
},
{
URL: "",
Method: "HEAD",
Status: http.StatusOK,
Golden: "testdata/golden/indexhead.txt",
},
{
URL: "one%25.txt",
Method: "HEAD",
Status: http.StatusOK,
Golden: "testdata/golden/onehead.txt",
},
{
URL: "",
Method: "POST",
Status: http.StatusMethodNotAllowed,
Golden: "testdata/golden/indexpost.txt",
},
{
URL: "one%25.txt",
Method: "POST",
Status: http.StatusMethodNotAllowed,
Golden: "testdata/golden/onepost.txt",
},
{
URL: "two.txt",
Status: http.StatusOK,
Golden: "testdata/golden/two.txt",
},
{
URL: "two.txt",
Status: http.StatusPartialContent,
Range: "bytes=2-5",
Golden: "testdata/golden/two2-5.txt",
},
{
URL: "two.txt",
Status: http.StatusPartialContent,
Range: "bytes=0-6",
Golden: "testdata/golden/two-6.txt",
},
{
URL: "two.txt",
Status: http.StatusPartialContent,
Range: "bytes=3-",
Golden: "testdata/golden/two3-.txt",
},
} {
method := test.Method
if method == "" {
method = "GET"
}
req, err := http.NewRequest(method, testURL+test.URL, nil)
require.NoError(t, err)
if test.Range != "" {
req.Header.Add("Range", test.Range)
}
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
assert.Equal(t, test.Status, resp.StatusCode, test.Golden)
body, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
checkGolden(t, test.Golden, body)
}
}
type mockNode struct {
path string
isdir bool
}
func (n mockNode) Path() string { return n.path }
func (n mockNode) Name() string {
if n.path == "" {
return ""
}
return path.Base(n.path)
}
func (n mockNode) IsDir() bool { return n.isdir }
func TestAddEntry(t *testing.T) {
var es entries
es.addEntry(mockNode{path: "", isdir: true})
es.addEntry(mockNode{path: "dir", isdir: true})
es.addEntry(mockNode{path: "a/b/c/d.txt", isdir: false})
es.addEntry(mockNode{path: "a/b/c/colon:colon.txt", isdir: false})
es.addEntry(mockNode{path: "\"quotes\".txt", isdir: false})
assert.Equal(t, entries{
{remote: "", URL: "/", Leaf: "/"},
{remote: "dir", URL: "dir/", Leaf: "dir/"},
{remote: "a/b/c/d.txt", URL: "d.txt", Leaf: "d.txt"},
{remote: "a/b/c/colon:colon.txt", URL: "./colon:colon.txt", Leaf: "colon:colon.txt"},
{remote: "\"quotes\".txt", URL: "%22quotes%22.txt", Leaf: "\"quotes\".txt"},
}, es)
}
func TestFinalise(t *testing.T) {
httpServer.srv.Close()
}

1
.rclone_repo/cmd/serve/http/testdata/files/hidden.txt vendored Executable file
View File

@@ -0,0 +1 @@
hidden

1
.rclone_repo/cmd/serve/http/testdata/files/hidden/file.txt vendored Executable file
View File

@@ -0,0 +1 @@
hiddenfile

1
.rclone_repo/cmd/serve/http/testdata/files/one%.txt vendored Executable file
View File

@@ -0,0 +1 @@
one%

1
.rclone_repo/cmd/serve/http/testdata/files/three/a.txt vendored Executable file
View File

@@ -0,0 +1 @@
three

1
.rclone_repo/cmd/serve/http/testdata/files/three/b.txt vendored Executable file
View File

@@ -0,0 +1 @@
threeb

1
.rclone_repo/cmd/serve/http/testdata/files/two.txt vendored Executable file
View File

@@ -0,0 +1 @@
0123456789

1
.rclone_repo/cmd/serve/http/testdata/golden/a.txt vendored Executable file
View File

@@ -0,0 +1 @@
three

1
.rclone_repo/cmd/serve/http/testdata/golden/dirnotfound.html vendored Executable file
View File

@@ -0,0 +1 @@
Directory not found

1
.rclone_repo/cmd/serve/http/testdata/golden/hidden.txt vendored Executable file
View File

@@ -0,0 +1 @@
File not found

1
.rclone_repo/cmd/serve/http/testdata/golden/hiddendir.html vendored Executable file
View File

@@ -0,0 +1 @@
Directory not found

13
.rclone_repo/cmd/serve/http/testdata/golden/index.html vendored Executable file
View File

@@ -0,0 +1,13 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Directory listing of /</title>
</head>
<body>
<h1>Directory listing of /</h1>
<a href="one%25.txt">one%.txt</a><br />
<a href="three/">three/</a><br />
<a href="two.txt">two.txt</a><br />
</body>
</html>

0
.rclone_repo/cmd/serve/http/testdata/golden/indexhead.txt vendored Executable file
View File

1
.rclone_repo/cmd/serve/http/testdata/golden/indexpost.txt vendored Executable file
View File

@@ -0,0 +1 @@
Method not allowed

1
.rclone_repo/cmd/serve/http/testdata/golden/notfound.html vendored Executable file
View File

@@ -0,0 +1 @@
File not found

1
.rclone_repo/cmd/serve/http/testdata/golden/one.txt vendored Executable file
View File

@@ -0,0 +1 @@
one%

0
.rclone_repo/cmd/serve/http/testdata/golden/onehead.txt vendored Executable file
View File

1
.rclone_repo/cmd/serve/http/testdata/golden/onepost.txt vendored Executable file
View File

@@ -0,0 +1 @@
Method not allowed

12
.rclone_repo/cmd/serve/http/testdata/golden/three.html vendored Executable file
View File

@@ -0,0 +1,12 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Directory listing of /three</title>
</head>
<body>
<h1>Directory listing of /three</h1>
<a href="a.txt">a.txt</a><br />
<a href="b.txt">b.txt</a><br />
</body>
</html>

1
.rclone_repo/cmd/serve/http/testdata/golden/two-6.txt vendored Executable file
View File

@@ -0,0 +1 @@
0123456

1
.rclone_repo/cmd/serve/http/testdata/golden/two.txt vendored Executable file
View File

@@ -0,0 +1 @@
0123456789

1
.rclone_repo/cmd/serve/http/testdata/golden/two2-5.txt vendored Executable file
View File

@@ -0,0 +1 @@
2345

1
.rclone_repo/cmd/serve/http/testdata/golden/two3-.txt vendored Executable file
View File

@@ -0,0 +1 @@
3456789