lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #173 from hawser/download-from-metadata

Browse Source 
Download from metadata
This commit is contained in:
risk danger olson 2015-03-05 11:30:40 -07:00
commit cb666d2140
4 changed files with 207 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

198
hawser/client.go
View File

@ -14,37 +14,22 @@ import (
"net/http"
"os"
"path/filepath"
"strings"
)
const (
gitMediaType = "application/vnd.git-media"
gitMediaMetaType = gitMediaType + "+json; charset=utf-8"
// Legacy type
gitMediaType = "application/vnd.git-media"
// The main type, sub type, and suffix. Use this when ensuring the type from
// an HTTP response is correct.
gitMediaMetaTypePrefix = gitMediaType + "+json"
// Adds the extra mime params. Use this when sending the type in an HTTP
// request.
gitMediaMetaType = gitMediaMetaTypePrefix + "; charset=utf-8"
)
type linkMeta struct {
Links map[string]*link `json:"_links,omitempty"`
}
func (l *linkMeta) Rel(name string) (*link, bool) {
if l.Links == nil {
return nil, false
}
lnk, ok := l.Links[name]
return lnk, ok
}
type link struct {
Href string `json:"href"`
Header map[string]string `json:"header,omitempty"`
}
type UploadRequest struct {
OidPath string
Filename string
CopyCallback CopyCallback
}
func Download(oidPath string) (io.ReadCloser, int64, *WrappedError) {
oid := filepath.Base(oidPath)
req, creds, err := request("GET", oid)
@ -66,6 +51,47 @@ func Download(oidPath string) (io.ReadCloser, int64, *WrappedError) {
return nil, 0, wErr
}
if strings.HasPrefix(contentType, gitMediaMetaTypePrefix) {
obj := &objectResource{}
err := json.NewDecoder(res.Body).Decode(obj)
res.Body.Close()
if err != nil {
wErr := Error(err)
setErrorResponseContext(wErr, res)
return nil, 0, wErr
}
dlReq, err := obj.NewRequest("download", "GET")
if err != nil {
wErr := Error(err)
setErrorResponseContext(wErr, res)
return nil, 0, wErr
}
dlCreds, err := setRequestHeaders(dlReq)
if err != nil {
return nil, 0, Errorf(err, "Error attempting to GET %s", oidPath)
}
dlRes, err := DoHTTP(Config, dlReq)
if err != nil {
wErr := Error(err)
setErrorResponseContext(wErr, res)
return nil, 0, wErr
}
saveCredentials(dlCreds, dlRes)
contentType := dlRes.Header.Get("Content-Type")
if contentType == "" {
wErr = Error(errors.New("Empty Content-Type"))
setErrorResponseContext(wErr, res)
return nil, 0, wErr
}
res = dlRes
}
ok, headerSize, wErr := validateMediaHeader(contentType, res.Body)
if !ok {
setErrorResponseContext(wErr, res)
@ -111,6 +137,46 @@ func Upload(oidPath, filename string, cb CopyCallback) *WrappedError {
return nil
}
type objectResource struct {
Oid string `json:"oid,omitempty"`
Size int64 `json:"size,omitempty"`
Links map[string]*linkRelation `json:"_links,omitempty"`
}
var objectRelationDoesNotExist = errors.New("relation does not exist")
func (o *objectResource) NewRequest(relation, method string) (*http.Request, error) {
rel, ok := o.Rel(relation)
if !ok {
return nil, objectRelationDoesNotExist
}
req, err := http.NewRequest(method, rel.Href, nil)
if err != nil {
return nil, err
}
for h, v := range rel.Header {
req.Header.Set(h, v)
}
return req, nil
}
func (o *objectResource) Rel(name string) (*linkRelation, bool) {
if o.Links == nil {
return nil, false
}
rel, ok := o.Links[name]
return rel, ok
}
type linkRelation struct {
Href string `json:"href"`
Header map[string]string `json:"header,omitempty"`
}
func callOptions(filehash string) (int, *WrappedError) {
oid := filepath.Base(filehash)
_, err := os.Stat(filehash)
@ -180,18 +246,22 @@ func callPut(filehash, filename string, cb CopyCallback) *WrappedError {
return wErr
}
func callExternalPut(filehash, filename string, lm *linkMeta, cb CopyCallback) *WrappedError {
if lm == nil {
func callExternalPut(filehash, filename string, obj *objectResource, cb CopyCallback) *WrappedError {
if obj == nil {
return Errorf(errors.New("No hypermedia links provided"),
"Error attempting to PUT %s", filename)
}
link, ok := lm.Rel("upload")
if !ok {
req, err := obj.NewRequest("upload", "PUT")
if err == objectRelationDoesNotExist {
return Errorf(errors.New("No upload link provided"),
"Error attempting to PUT %s", filename)
}
if err != nil {
return Errorf(err, "Error attempting to PUT %s", filename)
}
file, err := os.Open(filehash)
if err != nil {
return Errorf(err, "Error attempting to PUT %s", filename)
@ -209,14 +279,6 @@ func callExternalPut(filehash, filename string, lm *linkMeta, cb CopyCallback) *
Reader: file,
}
req, err := http.NewRequest("PUT", link.Href, nil)
if err != nil {
return Errorf(err, "Error attempting to PUT %s", filename)
}
for h, v := range link.Header {
req.Header.Set(h, v)
}
creds, err := setRequestHeaders(req)
if err != nil {
return Errorf(err, "Error attempting to PUT %s", filename)
@ -238,39 +300,36 @@ func callExternalPut(filehash, filename string, lm *linkMeta, cb CopyCallback) *
saveCredentials(creds, res)
// Run the verify callback
if cb, ok := lm.Rel("verify"); ok {
oid := filepath.Base(filehash)
verifyReq, err := http.NewRequest("POST", cb.Href, nil)
if err != nil {
return Errorf(err, "Error attempting to verify %s", filename)
}
for h, v := range cb.Header {
verifyReq.Header.Set(h, v)
}
verifyCreds, err := setRequestHeaders(verifyReq)
if err != nil {
return Errorf(err, "Error attempting to verify %s", filename)
}
d := fmt.Sprintf(`{"oid":"%s", "size":%d}`, oid, fileSize)
verifyReq.Body = ioutil.NopCloser(bytes.NewBufferString(d))
tracerx.Printf("verify: %s %s", oid, cb.Href)
verifyRes, err := DoHTTP(Config, verifyReq)
if err != nil {
return Errorf(err, "Error attempting to verify %s", filename)
}
tracerx.Printf("verify_status: %d", verifyRes.StatusCode)
saveCredentials(verifyCreds, verifyRes)
verifyReq, err := obj.NewRequest("verify", "POST")
if err == objectRelationDoesNotExist {
return nil
}
if err != nil {
return Errorf(err, "Error attempting to verify %s", filename)
}
verifyCreds, err := setRequestHeaders(verifyReq)
if err != nil {
return Errorf(err, "Error attempting to verify %s", filename)
}
oid := filepath.Base(filehash)
d := fmt.Sprintf(`{"oid":"%s", "size":%d}`, oid, fileSize)
verifyReq.Body = ioutil.NopCloser(bytes.NewBufferString(d))
tracerx.Printf("verify: %s %s", oid, verifyReq.URL.String())
verifyRes, err := DoHTTP(Config, verifyReq)
if err != nil {
return Errorf(err, "Error attempting to verify %s", filename)
}
tracerx.Printf("verify_status: %d", verifyRes.StatusCode)
saveCredentials(verifyCreds, verifyRes)
return nil
}
func callPost(filehash, filename string) (*linkMeta, int, *WrappedError) {
func callPost(filehash, filename string) (*objectResource, int, *WrappedError) {
oid := filepath.Base(filehash)
req, creds, err := request("POST", "")
if err != nil {
@ -302,14 +361,13 @@ func callPost(filehash, filename string) (*linkMeta, int, *WrappedError) {
tracerx.Printf("api_post_status: %d", res.StatusCode)
if res.StatusCode == 202 {
lm := &linkMeta{}
dec := json.NewDecoder(res.Body)
err := dec.Decode(lm)
obj := &objectResource{}
err := json.NewDecoder(res.Body).Decode(obj)
if err != nil {
return nil, res.StatusCode, Errorf(err, "Error decoding JSON from %s %s.", req.Method, req.URL)
}
return lm, res.StatusCode, nil
return obj, res.StatusCode, nil
}
return nil, res.StatusCode, nil

72
hawser/client_download_test.go
View File

@ -1,6 +1,7 @@
package hawser
import (
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
@ -49,6 +50,77 @@ func TestDownload(t *testing.T) {
}
}
func TestDownloadFromMeta(t *testing.T) {
mux := http.NewServeMux()
server := httptest.NewServer(mux)
tmp := tempdir(t)
defer server.Close()
defer os.RemoveAll(tmp)
// simulates an endpoint that returns the meta data for every request.
// this way downloads keep working with the older prototype server during
// the pre-release.
mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
w.WriteHeader(405)
return
}
obj := &objectResource{
Oid: "oid",
Size: 4,
Links: map[string]*linkRelation{
"download": &linkRelation{
Href: server.URL + "/media/download/oid",
},
},
}
by, err := json.Marshal(obj)
if err != nil {
t.Errorf("Error marshaling json: %s", err)
}
head := w.Header()
head.Set("Content-Type", "application/vnd.git-media+json")
w.WriteHeader(200)
w.Write(by)
})
mux.HandleFunc("/media/download/oid", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
w.WriteHeader(405)
return
}
head := w.Header()
head.Set("Content-Type", "application/octet-stream")
head.Set("Content-Length", "4")
w.WriteHeader(200)
w.Write([]byte("test"))
})
Config.SetConfig("hawser.url", server.URL+"/media")
reader, size, wErr := Download("whatever/oid")
if wErr != nil {
t.Fatalf("unexpected error: %s", wErr)
}
defer reader.Close()
if size != 4 {
t.Errorf("unexpected size: %d", size)
}
by, err := ioutil.ReadAll(reader)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if body := string(by); body != "test" {
t.Errorf("unexpected body: %s", body)
}
}
func TestDownloadWithRedirect(t *testing.T) {
mux := http.NewServeMux()
server := httptest.NewServer(mux)

6
hawser/client_test.go
View File

@ -88,8 +88,8 @@ func TestExternalPut(t *testing.T) {
w.WriteHeader(200)
})
link := &linkMeta{
Links: map[string]*link{
obj := &objectResource{
Links: map[string]*linkRelation{
"upload": {
Href: server.URL + "/media/objects/oid",
Header: map[string]string{"a": "1"},
@ -101,7 +101,7 @@ func TestExternalPut(t *testing.T) {
},
}
if err := callExternalPut(oidPath, "", link, nil); err != nil {
if err := callExternalPut(oidPath, "", obj, nil); err != nil {
t.Error(err)
}

8
hawser/client_upload_test.go
View File

@ -43,8 +43,8 @@ func TestUploadWithVerify(t *testing.T) {
posted = true
link := &linkMeta{
Links: map[string]*link{
obj := &objectResource{
Links: map[string]*linkRelation{
"upload": {
Href: server.URL + "/media/objects/oid",
Header: map[string]string{"a": "1"},
@ -56,9 +56,9 @@ func TestUploadWithVerify(t *testing.T) {
},
}
by, err := json.Marshal(link)
by, err := json.Marshal(obj)
if err != nil {
t.Errorf("Error marshaling link json: %s", link)
t.Errorf("Error marshaling link json: %s", obj)
w.WriteHeader(500)
return
}