diff --git a/docs/api.md b/docs/api.md index dd1a68e1..cd7350b3 100644 --- a/docs/api.md +++ b/docs/api.md @@ -7,8 +7,9 @@ Git repositories that use Git LFS will specify a URI endpoint. See the Use that endpoint as a base, and append the following relative paths to upload and download from the Git LFS server. -All requests should send an Accept header of `application/vnd.git-lfs+json`. -This may change in the future as the API evolves. +API requests require an Accept header of `application/vnd.git-lfs+json`. The +upload and verify requests need a `application/vnd.git-lfs+json` Content-Type +too. ## API Responses @@ -153,7 +154,7 @@ This request initiates the upload of an object, given a JSON body with the oid and size of the object to upload. ``` -> POST https://git-lfs-server.com/objects/ HTTP/1.1 +> POST https://git-lfs-server.com/objects HTTP/1.1 > Accept: application/vnd.git-lfs+json > Content-Type: application/vnd.git-lfs+json > Authorization: Basic ... (if authentication is needed) @@ -215,7 +216,7 @@ API expects a POST to the href after a successful upload. Git LFS clients send: ``` > POST https://git-lfs-server.com/callback -> Accept: application/vnd.git-lfs +> Accept: application/vnd.git-lfs+json > Content-Type: application/vnd.git-lfs+json > Content-Length: 123 > diff --git a/lfs/client.go b/lfs/client.go index b0a189cc..2d2982f4 100644 --- a/lfs/client.go +++ b/lfs/client.go @@ -1,6 +1,7 @@ package lfs import ( + "bytes" "encoding/base64" "encoding/json" "errors" @@ -8,7 +9,10 @@ import ( "io" "io/ioutil" "net/http" + "os" + "path/filepath" "regexp" + "strconv" ) const ( @@ -37,22 +41,22 @@ type objectResource struct { Links map[string]*linkRelation `json:"_links,omitempty"` } -func (o *objectResource) NewRequest(relation, method string) (*http.Request, error) { +func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) { rel, ok := o.Rel(relation) if !ok { - return nil, objectRelationDoesNotExist + return nil, nil, objectRelationDoesNotExist } - req, err := http.NewRequest(method, rel.Href, nil) + req, creds, err := newClientRequest(method, rel.Href) if err != nil { - return nil, err + return nil, nil, err } for h, v := range rel.Header { req.Header.Set(h, v) } - return req, nil + return req, creds, nil } func (o *objectResource) Rel(name string) (*linkRelation, bool) { @@ -87,14 +91,105 @@ func (e *ClientError) Error() string { } func Download(oid string) (io.ReadCloser, int64, *WrappedError) { - return nil, 0, nil + req, creds, err := newApiRequest("GET", oid) + if err != nil { + return nil, 0, Error(err) + } + + res, obj, wErr := doApiRequest(req, creds) + if wErr != nil { + return nil, 0, wErr + } + + req, creds, err = obj.NewRequest("download", "GET") + if err != nil { + return nil, 0, Error(err) + } + + res, wErr = doHttpRequest(req, creds) + if wErr != nil { + return nil, 0, wErr + } + + return res.Body, res.ContentLength, nil } -func Upload(oid, filename string, cb CopyCallback) *WrappedError { - return nil +func Upload(oidPath, filename string, cb CopyCallback) *WrappedError { + oid := filepath.Base(oidPath) + file, err := os.Open(oidPath) + if err != nil { + return Error(err) + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + return Error(err) + } + + reqObj := &objectResource{ + Oid: oid, + Size: stat.Size(), + } + + by, err := json.Marshal(reqObj) + if err != nil { + return Error(err) + } + + req, creds, err := newApiRequest("POST", "") + if err != nil { + return Error(err) + } + + req.Header.Set("Content-Type", mediaType) + req.Header.Set("Content-Length", strconv.Itoa(len(by))) + req.Body = ioutil.NopCloser(bytes.NewReader(by)) + + res, obj, wErr := doApiRequest(req, creds) + if wErr != nil { + return wErr + } + + req, creds, err = obj.NewRequest("upload", "PUT") + if err != nil { + return Error(err) + } + + if len(req.Header.Get("Content-Type")) == 0 { + req.Header.Set("Content-Type", "application/octet-stream") + } + req.Header.Set("Content-Length", strconv.FormatInt(reqObj.Size, 10)) + fmt.Println(req.Header) + req.Body = file + + res, wErr = doHttpRequest(req, creds) + if wErr != nil { + return wErr + } + + if res.StatusCode > 299 { + return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode) + } + + req, creds, err = obj.NewRequest("verify", "POST") + if err == objectRelationDoesNotExist { + return nil + } else if err != nil { + return Error(err) + } + + req.Header.Set("Content-Type", mediaType) + req.Header.Set("Content-Length", strconv.Itoa(len(by))) + req.Body = ioutil.NopCloser(bytes.NewReader(by)) + _, wErr = doHttpRequest(req, creds) + + return wErr } -func doApiRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) { +func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) { + fmt.Printf("HTTP %s %s\n", req.Method, req.URL) + fmt.Printf("HTTP HEADER: %v\n", req.Header) res, err := DoHTTP(Config, req) var wErr *WrappedError @@ -102,6 +197,7 @@ func doApiRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError if err != nil { wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL) } else { + fmt.Printf("HTTP Status %d\n", res.StatusCode) if creds != nil { saveCredentials(creds, res) } @@ -120,6 +216,22 @@ func doApiRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError return res, wErr } +func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) { + res, wErr := doHttpRequest(req, creds) + if wErr != nil { + return res, nil, wErr + } + + obj := &objectResource{} + wErr = decodeApiResponse(res, obj) + + if wErr != nil { + setErrorResponseContext(wErr, res) + } + + return res, obj, wErr +} + func handleResponse(res *http.Response) *WrappedError { if res.StatusCode < 400 { return nil @@ -130,24 +242,33 @@ func handleResponse(res *http.Response) *WrappedError { res.Body.Close() }() - var wErr *WrappedError - - if mediaTypeRE.MatchString(res.Header.Get("Content-Type")) { - cliErr := &ClientError{} - err := json.NewDecoder(res.Body).Decode(cliErr) - if err != nil { - return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL) + cliErr := &ClientError{} + wErr := decodeApiResponse(res, cliErr) + if wErr == nil { + if len(cliErr.Message) == 0 { + wErr = defaultError(res) + } else { + wErr = Error(cliErr) } - - wErr = Error(cliErr) - } else { - wErr = defaultError(res) } wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509 return wErr } +func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError { + if !mediaTypeRE.MatchString(res.Header.Get("Content-Type")) { + return nil + } + + err := json.NewDecoder(res.Body).Decode(obj) + if err != nil { + return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL) + } + + return nil +} + func defaultError(res *http.Response) *WrappedError { var msgFmt string diff --git a/lfs/download_test.go b/lfs/download_test.go new file mode 100644 index 00000000..2ba561ce --- /dev/null +++ b/lfs/download_test.go @@ -0,0 +1,97 @@ +package lfs + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "strconv" + "testing" +) + +func TestSuccessfulDownload(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + tmp := tempdir(t) + defer server.Close() + defer os.RemoveAll(tmp) + + mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) { + t.Logf("Method: %s", r.Method) + + if r.Method != "GET" { + w.WriteHeader(405) + return + } + + if accept := r.Header.Get("Accept"); accept != mediaType { + t.Errorf("Invalid Accept: %s", accept) + } + + obj := &objectResource{ + Oid: "oid", + Size: 4, + Links: map[string]*linkRelation{ + "download": &linkRelation{ + Href: server.URL + "/download", + Header: map[string]string{"A": "1"}, + }, + }, + } + + by, err := json.Marshal(obj) + if err != nil { + t.Fatal(err) + } + + head := w.Header() + head.Set("Content-Type", mediaType) + head.Set("Content-Length", strconv.Itoa(len(by))) + w.WriteHeader(200) + w.Write(by) + }) + + mux.HandleFunc("/download", func(w http.ResponseWriter, r *http.Request) { + t.Logf("Method: %s", r.Method) + + if r.Method != "GET" { + w.WriteHeader(405) + return + } + + if accept := r.Header.Get("Accept"); accept != "" { + t.Errorf("Accept: %s", accept) + } + + if a := r.Header.Get("A"); a != "1" { + t.Logf("A: %s", a) + } + + head := w.Header() + head.Set("Content-Type", "application/octet-stream") + head.Set("Content-Length", "4") + w.WriteHeader(200) + w.Write([]byte("test")) + }) + + Config.SetConfig("lfs.url", server.URL+"/media") + reader, size, wErr := Download("oid") + if wErr != nil { + t.Fatalf("unexpected error: %s", wErr) + } + defer reader.Close() + + if size != 4 { + t.Errorf("unexpected size: %d", size) + } + + by, err := ioutil.ReadAll(reader) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + if body := string(by); body != "test" { + t.Errorf("unexpected body: %s", body) + } +} diff --git a/lfs/upload_test.go b/lfs/upload_test.go new file mode 100644 index 00000000..c0a2844b --- /dev/null +++ b/lfs/upload_test.go @@ -0,0 +1,299 @@ +package lfs + +import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "testing" +) + +func TestSuccessfulUploadWithVerify(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + tmp := tempdir(t) + defer server.Close() + defer os.RemoveAll(tmp) + + postCalled := false + putCalled := false + verifyCalled := false + + mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) { + t.Logf("Server: %s %s", r.Method, r.URL) + + if r.Method != "POST" { + w.WriteHeader(405) + return + } + + if r.Header.Get("Accept") != mediaType { + t.Errorf("Invalid Accept") + } + + if r.Header.Get("Content-Type") != mediaType { + t.Errorf("Invalid Content-Type") + } + + buf := &bytes.Buffer{} + tee := io.TeeReader(r.Body, buf) + reqObj := &objectResource{} + err := json.NewDecoder(tee).Decode(reqObj) + t.Logf("request header: %v", r.Header) + t.Logf("request body: %s", buf.String()) + if err != nil { + t.Fatal(err) + } + + if reqObj.Oid != "oid" { + t.Errorf("invalid oid from request: %s", reqObj.Oid) + } + + if reqObj.Size != 4 { + t.Errorf("invalid size from request: %d", reqObj.Size) + } + + obj := &objectResource{ + Links: map[string]*linkRelation{ + "upload": &linkRelation{ + Href: server.URL + "/upload", + Header: map[string]string{"A": "1"}, + }, + "verify": &linkRelation{ + Href: server.URL + "/verify", + Header: map[string]string{"B": "2"}, + }, + }, + } + + by, err := json.Marshal(obj) + if err != nil { + t.Fatal(err) + } + + postCalled = true + head := w.Header() + head.Set("Content-Type", mediaType) + head.Set("Content-Length", strconv.Itoa(len(by))) + w.WriteHeader(200) + w.Write(by) + }) + + mux.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) { + t.Logf("Server: %s %s", r.Method, r.URL) + + if r.Method != "PUT" { + w.WriteHeader(405) + return + } + + if r.Header.Get("A") != "1" { + t.Error("Invalid A") + } + + if r.Header.Get("Content-Type") != "application/octet-stream" { + t.Error("Invalid Content-Type") + } + + by, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + + t.Logf("request header: %v", r.Header) + t.Logf("request body: %s", string(by)) + + if str := string(by); str != "test" { + t.Errorf("unexpected body: %s", str) + } + + putCalled = true + w.WriteHeader(200) + }) + + mux.HandleFunc("/verify", func(w http.ResponseWriter, r *http.Request) { + t.Logf("Server: %s %s", r.Method, r.URL) + + if r.Method != "POST" { + w.WriteHeader(405) + return + } + + if r.Header.Get("B") != "2" { + t.Error("Invalid B") + } + + if r.Header.Get("Content-Type") != mediaType { + t.Error("Invalid Content-Type") + } + + buf := &bytes.Buffer{} + tee := io.TeeReader(r.Body, buf) + reqObj := &objectResource{} + err := json.NewDecoder(tee).Decode(reqObj) + t.Logf("request header: %v", r.Header) + t.Logf("request body: %s", buf.String()) + if err != nil { + t.Fatal(err) + } + + if reqObj.Oid != "oid" { + t.Errorf("invalid oid from request: %s", reqObj.Oid) + } + + if reqObj.Size != 4 { + t.Errorf("invalid size from request: %d", reqObj.Size) + } + + verifyCalled = true + w.WriteHeader(200) + }) + + Config.SetConfig("lfs.url", server.URL+"/media") + + oidPath := filepath.Join(tmp, "oid") + if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil { + t.Fatal(err) + } + + wErr := Upload(oidPath, "", nil) + if wErr != nil { + t.Fatal(wErr) + } + + if !postCalled { + t.Errorf("POST not called") + } + + if !putCalled { + t.Errorf("PUT not called") + } + + if !verifyCalled { + t.Errorf("verify not called") + } +} + +func TestSuccessfulUploadWithoutVerify(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + tmp := tempdir(t) + defer server.Close() + defer os.RemoveAll(tmp) + + postCalled := false + putCalled := false + + mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) { + t.Logf("Server: %s %s", r.Method, r.URL) + + if r.Method != "POST" { + w.WriteHeader(405) + return + } + + if r.Header.Get("Accept") != mediaType { + t.Errorf("Invalid Accept") + } + + if r.Header.Get("Content-Type") != mediaType { + t.Errorf("Invalid Content-Type") + } + + buf := &bytes.Buffer{} + tee := io.TeeReader(r.Body, buf) + reqObj := &objectResource{} + err := json.NewDecoder(tee).Decode(reqObj) + t.Logf("request header: %v", r.Header) + t.Logf("request body: %s", buf.String()) + if err != nil { + t.Fatal(err) + } + + if reqObj.Oid != "oid" { + t.Errorf("invalid oid from request: %s", reqObj.Oid) + } + + if reqObj.Size != 4 { + t.Errorf("invalid size from request: %d", reqObj.Size) + } + + obj := &objectResource{ + Links: map[string]*linkRelation{ + "upload": &linkRelation{ + Href: server.URL + "/upload", + Header: map[string]string{"A": "1"}, + }, + }, + } + + by, err := json.Marshal(obj) + if err != nil { + t.Fatal(err) + } + + postCalled = true + head := w.Header() + head.Set("Content-Type", mediaType) + head.Set("Content-Length", strconv.Itoa(len(by))) + w.WriteHeader(200) + w.Write(by) + }) + + mux.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) { + t.Logf("Server: %s %s", r.Method, r.URL) + + if r.Method != "PUT" { + w.WriteHeader(405) + return + } + + if a := r.Header.Get("A"); a != "1" { + t.Errorf("Invalid A: %s", a) + } + + if r.Header.Get("Content-Type") != "application/octet-stream" { + t.Error("Invalid Content-Type") + } + + by, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + + t.Logf("request header: %v", r.Header) + t.Logf("request body: %s", string(by)) + + if str := string(by); str != "test" { + t.Errorf("unexpected body: %s", str) + } + + putCalled = true + w.WriteHeader(200) + }) + + Config.SetConfig("lfs.url", server.URL+"/media") + + oidPath := filepath.Join(tmp, "oid") + if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil { + t.Fatal(err) + } + + wErr := Upload(oidPath, "", nil) + if wErr != nil { + t.Fatal(wErr) + } + + if !postCalled { + t.Errorf("POST not called") + } + + if !putCalled { + t.Errorf("PUT not called") + } +}