overdue

.rclone_repo/backend/webdav/api/types.go

@@ -0,0 +1,164 @@
+// Package api has type definitions for webdav
+package api
+
+import (
+	"encoding/xml"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	// Wed, 27 Sep 2017 14:28:34 GMT
+	timeFormat = time.RFC1123
+)
+
+// Multistatus contains responses returned from an HTTP 207 return code
+type Multistatus struct {
+	Responses []Response `xml:"response"`
+}
+
+// Response contains an Href the response it about and its properties
+type Response struct {
+	Href  string `xml:"href"`
+	Props Prop   `xml:"propstat"`
+}
+
+// Prop is the properties of a response
+//
+// This is a lazy way of decoding the multiple <s:propstat> in the
+// response.
+//
+// The response might look like this
+//
+// <d:response>
+//   <d:href>/remote.php/webdav/Nextcloud%20Manual.pdf</d:href>
+//   <d:propstat>
+//     <d:prop>
+//       <d:getlastmodified>Tue, 19 Dec 2017 22:02:36 GMT</d:getlastmodified>
+//       <d:getcontentlength>4143665</d:getcontentlength>
+//       <d:resourcetype/>
+//       <d:getetag>"048d7be4437ff7deeae94db50ff3e209"</d:getetag>
+//       <d:getcontenttype>application/pdf</d:getcontenttype>
+//     </d:prop>
+//     <d:status>HTTP/1.1 200 OK</d:status>
+//   </d:propstat>
+//   <d:propstat>
+//     <d:prop>
+//       <d:quota-used-bytes/>
+//       <d:quota-available-bytes/>
+//     </d:prop>
+//     <d:status>HTTP/1.1 404 Not Found</d:status>
+//   </d:propstat>
+// </d:response>
+//
+// So we elide the array of <d:propstat> and within that the array of
+// <d:prop> into one struct.
+//
+// Note that status collects all the status values for which we just
+// check the first is OK.
+type Prop struct {
+	Status   []string  `xml:"DAV: status"`
+	Name     string    `xml:"DAV: prop>displayname,omitempty"`
+	Type     *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
+	Size     int64     `xml:"DAV: prop>getcontentlength,omitempty"`
+	Modified Time      `xml:"DAV: prop>getlastmodified,omitempty"`
+}
+
+// Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
+var parseStatus = regexp.MustCompile(`^HTTP/[0-9.]+\s+(\d+)`)
+
+// StatusOK examines the Status and returns an OK flag
+func (p *Prop) StatusOK() bool {
+	// Assume OK if no statuses received
+	if len(p.Status) == 0 {
+		return true
+	}
+	match := parseStatus.FindStringSubmatch(p.Status[0])
+	if len(match) < 2 {
+		return false
+	}
+	code, err := strconv.Atoi(match[1])
+	if err != nil {
+		return false
+	}
+	if code >= 200 && code < 300 {
+		return true
+	}
+	return false
+}
+
+// PropValue is a tagged name and value
+type PropValue struct {
+	XMLName xml.Name `xml:""`
+	Value   string   `xml:",chardata"`
+}
+
+// Error is used to desribe webdav errors
+//
+// <d:error xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns">
+//   <s:exception>Sabre\DAV\Exception\NotFound</s:exception>
+//   <s:message>File with name Photo could not be located</s:message>
+// </d:error>
+type Error struct {
+	Exception  string `xml:"exception,omitempty"`
+	Message    string `xml:"message,omitempty"`
+	Status     string
+	StatusCode int
+}
+
+// Error returns a string for the error and statistifes the error interface
+func (e *Error) Error() string {
+	var out []string
+	if e.Message != "" {
+		out = append(out, e.Message)
+	}
+	if e.Exception != "" {
+		out = append(out, e.Exception)
+	}
+	if e.Status != "" {
+		out = append(out, e.Status)
+	}
+	if len(out) == 0 {
+		return "Webdav Error"
+	}
+	return strings.Join(out, ": ")
+}
+
+// Time represents represents date and time information for the
+// webdav API marshalling to and from timeFormat
+type Time time.Time
+
+// MarshalXML turns a Time into XML
+func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	timeString := (*time.Time)(t).Format(timeFormat)
+	return e.EncodeElement(timeString, start)
+}
+
+// Possible time formats to parse the time with
+var timeFormats = []string{
+	timeFormat,    // Wed, 27 Sep 2017 14:28:34 GMT (as per RFC)
+	time.RFC1123Z, // Fri, 05 Jan 2018 14:14:38 +0000 (as used by mydrive.ch)
+	time.UnixDate, // Wed May 17 15:31:58 UTC 2017 (as used in an internal server)
+}
+
+// UnmarshalXML turns XML into a Time
+func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	var v string
+	err := d.DecodeElement(&v, &start)
+	if err != nil {
+		return err
+	}
+
+	// Parse the time format in multiple possible ways
+	var newT time.Time
+	for _, timeFormat := range timeFormats {
+		newT, err = time.Parse(timeFormat, v)
+		if err == nil {
+			*t = Time(newT)
+			break
+		}
+	}
+	return err
+}

.rclone_repo/backend/webdav/odrvcookie/fetch.go

@@ -0,0 +1,186 @@
+// Package odrvcookie can fetch authentication cookies for a sharepoint webdav endpoint
+package odrvcookie
+
+import (
+	"bytes"
+	"encoding/xml"
+	"html/template"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/fshttp"
+	"github.com/pkg/errors"
+	"golang.org/x/net/publicsuffix"
+)
+
+// CookieAuth hold the authentication information
+// These are username and password as well as the authentication endpoint
+type CookieAuth struct {
+	user     string
+	pass     string
+	endpoint string
+}
+
+// CookieResponse contains the requested cookies
+type CookieResponse struct {
+	RtFa    http.Cookie
+	FedAuth http.Cookie
+}
+
+// SuccessResponse hold a response from the sharepoint webdav
+type SuccessResponse struct {
+	XMLName xml.Name            `xml:"Envelope"`
+	Succ    SuccessResponseBody `xml:"Body"`
+}
+
+// SuccessResponseBody is the body of a success response, it holds the token
+type SuccessResponseBody struct {
+	XMLName xml.Name
+	Type    string    `xml:"RequestSecurityTokenResponse>TokenType"`
+	Created time.Time `xml:"RequestSecurityTokenResponse>Lifetime>Created"`
+	Expires time.Time `xml:"RequestSecurityTokenResponse>Lifetime>Expires"`
+	Token   string    `xml:"RequestSecurityTokenResponse>RequestedSecurityToken>BinarySecurityToken"`
+}
+
+// reqString is a template that gets populated with the user data in order to retrieve a "BinarySecurityToken"
+const reqString = `<s:Envelope xmlns:s="http://www.w3.org/2003/05/soap-envelope"
+xmlns:a="http://www.w3.org/2005/08/addressing"
+xmlns:u="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd">
+<s:Header>
+<a:Action s:mustUnderstand="1">http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue</a:Action>
+<a:ReplyTo>
+<a:Address>http://www.w3.org/2005/08/addressing/anonymous</a:Address>
+</a:ReplyTo>
+<a:To s:mustUnderstand="1">https://login.microsoftonline.com/extSTS.srf</a:To>
+<o:Security s:mustUnderstand="1"
+ xmlns:o="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd">
+<o:UsernameToken>
+  <o:Username>{{ .Username }}</o:Username>
+  <o:Password>{{ .Password }}</o:Password>
+</o:UsernameToken>
+</o:Security>
+</s:Header>
+<s:Body>
+<t:RequestSecurityToken xmlns:t="http://schemas.xmlsoap.org/ws/2005/02/trust">
+<wsp:AppliesTo xmlns:wsp="http://schemas.xmlsoap.org/ws/2004/09/policy">
+  <a:EndpointReference>
+    <a:Address>{{ .Address }}</a:Address>
+  </a:EndpointReference>
+</wsp:AppliesTo>
+<t:KeyType>http://schemas.xmlsoap.org/ws/2005/05/identity/NoProofKey</t:KeyType>
+<t:RequestType>http://schemas.xmlsoap.org/ws/2005/02/trust/Issue</t:RequestType>
+<t:TokenType>urn:oasis:names:tc:SAML:1.0:assertion</t:TokenType>
+</t:RequestSecurityToken>
+</s:Body>
+</s:Envelope>`
+
+// New creates a new CookieAuth struct
+func New(pUser, pPass, pEndpoint string) CookieAuth {
+	retStruct := CookieAuth{
+		user:     pUser,
+		pass:     pPass,
+		endpoint: pEndpoint,
+	}
+
+	return retStruct
+}
+
+// Cookies creates a CookieResponse. It fetches the auth token and then
+// retrieves the Cookies
+func (ca *CookieAuth) Cookies() (*CookieResponse, error) {
+	tokenResp, err := ca.getSPToken()
+	if err != nil {
+		return nil, err
+	}
+	return ca.getSPCookie(tokenResp)
+}
+
+func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error) {
+	spRoot, err := url.Parse(ca.endpoint)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error while contructing endpoint URL")
+	}
+
+	u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
+	if err != nil {
+		return nil, errors.Wrap(err, "Error while constructing login URL")
+	}
+
+	// To authenticate with davfs or anything else we need two cookies (rtFa and FedAuth)
+	// In order to get them we use the token we got earlier and a cookieJar
+	jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
+	if err != nil {
+		return nil, err
+	}
+
+	client := &http.Client{
+		Jar: jar,
+	}
+
+	// Send the previously aquired Token as a Post parameter
+	if _, err = client.Post(u.String(), "text/xml", strings.NewReader(conf.Succ.Token)); err != nil {
+		return nil, errors.Wrap(err, "Error while grabbing cookies from endpoint: %v")
+	}
+
+	cookieResponse := CookieResponse{}
+	for _, cookie := range jar.Cookies(u) {
+		if (cookie.Name == "rtFa") || (cookie.Name == "FedAuth") {
+			switch cookie.Name {
+			case "rtFa":
+				cookieResponse.RtFa = *cookie
+			case "FedAuth":
+				cookieResponse.FedAuth = *cookie
+			}
+		}
+	}
+	return &cookieResponse, nil
+}
+
+func (ca *CookieAuth) getSPToken() (conf *SuccessResponse, err error) {
+	reqData := map[string]interface{}{
+		"Username": ca.user,
+		"Password": ca.pass,
+		"Address":  ca.endpoint,
+	}
+
+	t := template.Must(template.New("authXML").Parse(reqString))
+
+	buf := &bytes.Buffer{}
+	if err := t.Execute(buf, reqData); err != nil {
+		return nil, errors.Wrap(err, "Error while filling auth token template")
+	}
+
+	// Create and execute the first request which returns an auth token for the sharepoint service
+	// With this token we can authenticate on the login page and save the returned cookies
+	req, err := http.NewRequest("POST", "https://login.microsoftonline.com/extSTS.srf", buf)
+	if err != nil {
+		return nil, err
+	}
+
+	client := fshttp.NewClient(fs.Config)
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error while logging in to endpoint")
+	}
+	defer fs.CheckClose(resp.Body, &err)
+
+	respBuf := bytes.Buffer{}
+	_, err = respBuf.ReadFrom(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	s := respBuf.Bytes()
+
+	conf = &SuccessResponse{}
+	err = xml.Unmarshal(s, conf)
+	if err != nil {
+		// FIXME: Try to parse with FailedResponse struct (check for server error code)
+		return nil, errors.Wrap(err, "Error while reading endpoint response")
+	}
+
+	return
+}

.rclone_repo/backend/webdav/webdav_test.go

@@ -0,0 +1,17 @@
+// Test Webdav filesystem interface
+package webdav_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/backend/webdav"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestWebdav:",
+		NilObject:  (*webdav.Object)(nil),
+	})
+}