implement uploads and downloads

docs/api.md

@@ -7,8 +7,9 @@ Git repositories that use Git LFS will specify a URI endpoint.  See the
 Use that endpoint as a base, and append the following relative paths to upload
 and download from the Git LFS server.
 
-All requests should send an Accept header of `application/vnd.git-lfs+json`.
+API requests require an Accept header of `application/vnd.git-lfs+json`. The
-This may change in the future as the API evolves.
+upload and verify requests need a `application/vnd.git-lfs+json` Content-Type
+too.
 
 ## API Responses
 
@@ -153,7 +154,7 @@ This request initiates the upload of an object, given a JSON body with the oid
 and size of the object to upload.
 
 ```
-> POST https://git-lfs-server.com/objects/ HTTP/1.1
+> POST https://git-lfs-server.com/objects HTTP/1.1
 > Accept: application/vnd.git-lfs+json
 > Content-Type: application/vnd.git-lfs+json
 > Authorization: Basic ... (if authentication is needed)
@@ -215,7 +216,7 @@ API expects a POST to the href after a successful upload.  Git LFS clients send:
 
 ```
 > POST https://git-lfs-server.com/callback
-> Accept: application/vnd.git-lfs
+> Accept: application/vnd.git-lfs+json
 > Content-Type: application/vnd.git-lfs+json
 > Content-Length: 123
 >

lfs/client.go

@@ -1,6 +1,7 @@
 package lfs
 
 import (
+	"bytes"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
@@ -8,7 +9,10 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"os"
+	"path/filepath"
 	"regexp"
+	"strconv"
 )
 
 const (
@@ -37,22 +41,22 @@ type objectResource struct {
 	Links map[string]*linkRelation `json:"_links,omitempty"`
 }
 
-func (o *objectResource) NewRequest(relation, method string) (*http.Request, error) {
+func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) {
 	rel, ok := o.Rel(relation)
 	if !ok {
-		return nil, objectRelationDoesNotExist
+		return nil, nil, objectRelationDoesNotExist
 	}
 
-	req, err := http.NewRequest(method, rel.Href, nil)
+	req, creds, err := newClientRequest(method, rel.Href)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	for h, v := range rel.Header {
 		req.Header.Set(h, v)
 	}
 
-	return req, nil
+	return req, creds, nil
 }
 
 func (o *objectResource) Rel(name string) (*linkRelation, bool) {
@@ -87,14 +91,105 @@ func (e *ClientError) Error() string {
 }
 
 func Download(oid string) (io.ReadCloser, int64, *WrappedError) {
-	return nil, 0, nil
+	req, creds, err := newApiRequest("GET", oid)
+	if err != nil {
+		return nil, 0, Error(err)
+	}
+
+	res, obj, wErr := doApiRequest(req, creds)
+	if wErr != nil {
+		return nil, 0, wErr
+	}
+
+	req, creds, err = obj.NewRequest("download", "GET")
+	if err != nil {
+		return nil, 0, Error(err)
+	}
+
+	res, wErr = doHttpRequest(req, creds)
+	if wErr != nil {
+		return nil, 0, wErr
+	}
+
+	return res.Body, res.ContentLength, nil
 }
 
-func Upload(oid, filename string, cb CopyCallback) *WrappedError {
+func Upload(oidPath, filename string, cb CopyCallback) *WrappedError {
+	oid := filepath.Base(oidPath)
+	file, err := os.Open(oidPath)
+	if err != nil {
+		return Error(err)
+	}
+	defer file.Close()
+
+	stat, err := file.Stat()
+	if err != nil {
+		return Error(err)
+	}
+
+	reqObj := &objectResource{
+		Oid:  oid,
+		Size: stat.Size(),
+	}
+
+	by, err := json.Marshal(reqObj)
+	if err != nil {
+		return Error(err)
+	}
+
+	req, creds, err := newApiRequest("POST", "")
+	if err != nil {
+		return Error(err)
+	}
+
+	req.Header.Set("Content-Type", mediaType)
+	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
+	req.Body = ioutil.NopCloser(bytes.NewReader(by))
+
+	res, obj, wErr := doApiRequest(req, creds)
+	if wErr != nil {
+		return wErr
+	}
+
+	req, creds, err = obj.NewRequest("upload", "PUT")
+	if err != nil {
+		return Error(err)
+	}
+
+	if len(req.Header.Get("Content-Type")) == 0 {
+		req.Header.Set("Content-Type", "application/octet-stream")
+	}
+	req.Header.Set("Content-Length", strconv.FormatInt(reqObj.Size, 10))
+	fmt.Println(req.Header)
+	req.Body = file
+
+	res, wErr = doHttpRequest(req, creds)
+	if wErr != nil {
+		return wErr
+	}
+
+	if res.StatusCode > 299 {
+		return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
+	}
+
+	req, creds, err = obj.NewRequest("verify", "POST")
+	if err == objectRelationDoesNotExist {
 		return nil
+	} else if err != nil {
+		return Error(err)
+	}
+
+	req.Header.Set("Content-Type", mediaType)
+	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
+	req.Body = ioutil.NopCloser(bytes.NewReader(by))
+	_, wErr = doHttpRequest(req, creds)
+
+	return wErr
 }
 
-func doApiRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
+func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
+	fmt.Printf("HTTP %s %s\n", req.Method, req.URL)
+	fmt.Printf("HTTP HEADER: %v\n", req.Header)
 	res, err := DoHTTP(Config, req)
 
 	var wErr *WrappedError
@@ -102,6 +197,7 @@ func doApiRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError
 	if err != nil {
 		wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL)
 	} else {
+		fmt.Printf("HTTP Status %d\n", res.StatusCode)
 		if creds != nil {
 			saveCredentials(creds, res)
 		}
@@ -120,6 +216,22 @@ func doApiRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError
 	return res, wErr
 }
 
+func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) {
+	res, wErr := doHttpRequest(req, creds)
+	if wErr != nil {
+		return res, nil, wErr
+	}
+
+	obj := &objectResource{}
+	wErr = decodeApiResponse(res, obj)
+
+	if wErr != nil {
+		setErrorResponseContext(wErr, res)
+	}
+
+	return res, obj, wErr
+}
+
 func handleResponse(res *http.Response) *WrappedError {
 	if res.StatusCode < 400 {
 		return nil
@@ -130,24 +242,33 @@ func handleResponse(res *http.Response) *WrappedError {
 		res.Body.Close()
 	}()
 
-	var wErr *WrappedError
-
-	if mediaTypeRE.MatchString(res.Header.Get("Content-Type")) {
 	cliErr := &ClientError{}
-		err := json.NewDecoder(res.Body).Decode(cliErr)
+	wErr := decodeApiResponse(res, cliErr)
-		if err != nil {
+	if wErr == nil {
-			return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
+		if len(cliErr.Message) == 0 {
-		}
-
-		wErr = Error(cliErr)
-	} else {
 			wErr = defaultError(res)
+		} else {
+			wErr = Error(cliErr)
+		}
 	}
 
 	wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509
 	return wErr
 }
 
+func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError {
+	if !mediaTypeRE.MatchString(res.Header.Get("Content-Type")) {
+		return nil
+	}
+
+	err := json.NewDecoder(res.Body).Decode(obj)
+	if err != nil {
+		return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
+	}
+
+	return nil
+}
+
 func defaultError(res *http.Response) *WrappedError {
 	var msgFmt string
 

lfs/download_test.go

@@ -0,0 +1,97 @@
+package lfs
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strconv"
+	"testing"
+)
+
+func TestSuccessfulDownload(t *testing.T) {
+	mux := http.NewServeMux()
+	server := httptest.NewServer(mux)
+	tmp := tempdir(t)
+	defer server.Close()
+	defer os.RemoveAll(tmp)
+
+	mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
+		t.Logf("Method: %s", r.Method)
+
+		if r.Method != "GET" {
+			w.WriteHeader(405)
+			return
+		}
+
+		if accept := r.Header.Get("Accept"); accept != mediaType {
+			t.Errorf("Invalid Accept: %s", accept)
+		}
+
+		obj := &objectResource{
+			Oid:  "oid",
+			Size: 4,
+			Links: map[string]*linkRelation{
+				"download": &linkRelation{
+					Href:   server.URL + "/download",
+					Header: map[string]string{"A": "1"},
+				},
+			},
+		}
+
+		by, err := json.Marshal(obj)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		head := w.Header()
+		head.Set("Content-Type", mediaType)
+		head.Set("Content-Length", strconv.Itoa(len(by)))
+		w.WriteHeader(200)
+		w.Write(by)
+	})
+
+	mux.HandleFunc("/download", func(w http.ResponseWriter, r *http.Request) {
+		t.Logf("Method: %s", r.Method)
+
+		if r.Method != "GET" {
+			w.WriteHeader(405)
+			return
+		}
+
+		if accept := r.Header.Get("Accept"); accept != "" {
+			t.Errorf("Accept: %s", accept)
+		}
+
+		if a := r.Header.Get("A"); a != "1" {
+			t.Logf("A: %s", a)
+		}
+
+		head := w.Header()
+		head.Set("Content-Type", "application/octet-stream")
+		head.Set("Content-Length", "4")
+		w.WriteHeader(200)
+		w.Write([]byte("test"))
+	})
+
+	Config.SetConfig("lfs.url", server.URL+"/media")
+	reader, size, wErr := Download("oid")
+	if wErr != nil {
+		t.Fatalf("unexpected error: %s", wErr)
+	}
+	defer reader.Close()
+
+	if size != 4 {
+		t.Errorf("unexpected size: %d", size)
+	}
+
+	by, err := ioutil.ReadAll(reader)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	if body := string(by); body != "test" {
+		t.Errorf("unexpected body: %s", body)
+	}
+}

lfs/upload_test.go

@@ -0,0 +1,299 @@
+package lfs
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+)
+
+func TestSuccessfulUploadWithVerify(t *testing.T) {
+	mux := http.NewServeMux()
+	server := httptest.NewServer(mux)
+	tmp := tempdir(t)
+	defer server.Close()
+	defer os.RemoveAll(tmp)
+
+	postCalled := false
+	putCalled := false
+	verifyCalled := false
+
+	mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) {
+		t.Logf("Server: %s %s", r.Method, r.URL)
+
+		if r.Method != "POST" {
+			w.WriteHeader(405)
+			return
+		}
+
+		if r.Header.Get("Accept") != mediaType {
+			t.Errorf("Invalid Accept")
+		}
+
+		if r.Header.Get("Content-Type") != mediaType {
+			t.Errorf("Invalid Content-Type")
+		}
+
+		buf := &bytes.Buffer{}
+		tee := io.TeeReader(r.Body, buf)
+		reqObj := &objectResource{}
+		err := json.NewDecoder(tee).Decode(reqObj)
+		t.Logf("request header: %v", r.Header)
+		t.Logf("request body: %s", buf.String())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if reqObj.Oid != "oid" {
+			t.Errorf("invalid oid from request: %s", reqObj.Oid)
+		}
+
+		if reqObj.Size != 4 {
+			t.Errorf("invalid size from request: %d", reqObj.Size)
+		}
+
+		obj := &objectResource{
+			Links: map[string]*linkRelation{
+				"upload": &linkRelation{
+					Href:   server.URL + "/upload",
+					Header: map[string]string{"A": "1"},
+				},
+				"verify": &linkRelation{
+					Href:   server.URL + "/verify",
+					Header: map[string]string{"B": "2"},
+				},
+			},
+		}
+
+		by, err := json.Marshal(obj)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		postCalled = true
+		head := w.Header()
+		head.Set("Content-Type", mediaType)
+		head.Set("Content-Length", strconv.Itoa(len(by)))
+		w.WriteHeader(200)
+		w.Write(by)
+	})
+
+	mux.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) {
+		t.Logf("Server: %s %s", r.Method, r.URL)
+
+		if r.Method != "PUT" {
+			w.WriteHeader(405)
+			return
+		}
+
+		if r.Header.Get("A") != "1" {
+			t.Error("Invalid A")
+		}
+
+		if r.Header.Get("Content-Type") != "application/octet-stream" {
+			t.Error("Invalid Content-Type")
+		}
+
+		by, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Error(err)
+		}
+
+		t.Logf("request header: %v", r.Header)
+		t.Logf("request body: %s", string(by))
+
+		if str := string(by); str != "test" {
+			t.Errorf("unexpected body: %s", str)
+		}
+
+		putCalled = true
+		w.WriteHeader(200)
+	})
+
+	mux.HandleFunc("/verify", func(w http.ResponseWriter, r *http.Request) {
+		t.Logf("Server: %s %s", r.Method, r.URL)
+
+		if r.Method != "POST" {
+			w.WriteHeader(405)
+			return
+		}
+
+		if r.Header.Get("B") != "2" {
+			t.Error("Invalid B")
+		}
+
+		if r.Header.Get("Content-Type") != mediaType {
+			t.Error("Invalid Content-Type")
+		}
+
+		buf := &bytes.Buffer{}
+		tee := io.TeeReader(r.Body, buf)
+		reqObj := &objectResource{}
+		err := json.NewDecoder(tee).Decode(reqObj)
+		t.Logf("request header: %v", r.Header)
+		t.Logf("request body: %s", buf.String())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if reqObj.Oid != "oid" {
+			t.Errorf("invalid oid from request: %s", reqObj.Oid)
+		}
+
+		if reqObj.Size != 4 {
+			t.Errorf("invalid size from request: %d", reqObj.Size)
+		}
+
+		verifyCalled = true
+		w.WriteHeader(200)
+	})
+
+	Config.SetConfig("lfs.url", server.URL+"/media")
+
+	oidPath := filepath.Join(tmp, "oid")
+	if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
+		t.Fatal(err)
+	}
+
+	wErr := Upload(oidPath, "", nil)
+	if wErr != nil {
+		t.Fatal(wErr)
+	}
+
+	if !postCalled {
+		t.Errorf("POST not called")
+	}
+
+	if !putCalled {
+		t.Errorf("PUT not called")
+	}
+
+	if !verifyCalled {
+		t.Errorf("verify not called")
+	}
+}
+
+func TestSuccessfulUploadWithoutVerify(t *testing.T) {
+	mux := http.NewServeMux()
+	server := httptest.NewServer(mux)
+	tmp := tempdir(t)
+	defer server.Close()
+	defer os.RemoveAll(tmp)
+
+	postCalled := false
+	putCalled := false
+
+	mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) {
+		t.Logf("Server: %s %s", r.Method, r.URL)
+
+		if r.Method != "POST" {
+			w.WriteHeader(405)
+			return
+		}
+
+		if r.Header.Get("Accept") != mediaType {
+			t.Errorf("Invalid Accept")
+		}
+
+		if r.Header.Get("Content-Type") != mediaType {
+			t.Errorf("Invalid Content-Type")
+		}
+
+		buf := &bytes.Buffer{}
+		tee := io.TeeReader(r.Body, buf)
+		reqObj := &objectResource{}
+		err := json.NewDecoder(tee).Decode(reqObj)
+		t.Logf("request header: %v", r.Header)
+		t.Logf("request body: %s", buf.String())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if reqObj.Oid != "oid" {
+			t.Errorf("invalid oid from request: %s", reqObj.Oid)
+		}
+
+		if reqObj.Size != 4 {
+			t.Errorf("invalid size from request: %d", reqObj.Size)
+		}
+
+		obj := &objectResource{
+			Links: map[string]*linkRelation{
+				"upload": &linkRelation{
+					Href:   server.URL + "/upload",
+					Header: map[string]string{"A": "1"},
+				},
+			},
+		}
+
+		by, err := json.Marshal(obj)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		postCalled = true
+		head := w.Header()
+		head.Set("Content-Type", mediaType)
+		head.Set("Content-Length", strconv.Itoa(len(by)))
+		w.WriteHeader(200)
+		w.Write(by)
+	})
+
+	mux.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) {
+		t.Logf("Server: %s %s", r.Method, r.URL)
+
+		if r.Method != "PUT" {
+			w.WriteHeader(405)
+			return
+		}
+
+		if a := r.Header.Get("A"); a != "1" {
+			t.Errorf("Invalid A: %s", a)
+		}
+
+		if r.Header.Get("Content-Type") != "application/octet-stream" {
+			t.Error("Invalid Content-Type")
+		}
+
+		by, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Error(err)
+		}
+
+		t.Logf("request header: %v", r.Header)
+		t.Logf("request body: %s", string(by))
+
+		if str := string(by); str != "test" {
+			t.Errorf("unexpected body: %s", str)
+		}
+
+		putCalled = true
+		w.WriteHeader(200)
+	})
+
+	Config.SetConfig("lfs.url", server.URL+"/media")
+
+	oidPath := filepath.Join(tmp, "oid")
+	if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
+		t.Fatal(err)
+	}
+
+	wErr := Upload(oidPath, "", nil)
+	if wErr != nil {
+		t.Fatal(wErr)
+	}
+
+	if !postCalled {
+		t.Errorf("POST not called")
+	}
+
+	if !putCalled {
+		t.Errorf("PUT not called")
+	}
+}