git-lfs/hawser/client.go

431 lines
9.6 KiB
Go
Raw Normal View History

package hawser
2013-10-04 17:09:03 +00:00
import (
2015-01-23 22:11:10 +00:00
"bytes"
"encoding/base64"
"encoding/json"
"errors"
2013-10-04 17:09:03 +00:00
"fmt"
"github.com/cheggaaa/pb"
2015-01-23 22:11:10 +00:00
"github.com/rubyist/tracerx"
"io"
"io/ioutil"
2014-04-16 14:30:13 +00:00
"mime"
2013-10-04 17:09:03 +00:00
"net/http"
"net/url"
2013-10-04 17:09:03 +00:00
"os"
"path"
"path/filepath"
2013-10-04 17:09:03 +00:00
)
const (
gitMediaType = "application/vnd.git-media"
2014-04-16 15:28:04 +00:00
gitMediaMetaType = gitMediaType + "+json; charset=utf-8"
)
2015-01-23 22:11:10 +00:00
type linkMeta struct {
Links map[string]*link `json:"_links,omitempty"`
}
type link struct {
Href string `json:"href"`
Header map[string]string `json:"header,omitempty"`
}
type UploadRequest struct {
OidPath string
Filename string
CopyCallback CopyCallback
}
type DownloadRequest struct {
OidPath string
}
func Download(download *DownloadRequest) (io.ReadCloser, int64, *WrappedError) {
oid := filepath.Base(download.OidPath)
req, creds, err := clientRequest("GET", oid)
if err != nil {
return nil, 0, Error(err)
}
req.Header.Set("Accept", gitMediaType)
res, wErr := doRequest(req, creds)
if wErr != nil {
return nil, 0, wErr
}
contentType := res.Header.Get("Content-Type")
if contentType == "" {
wErr = Error(errors.New("Empty Content-Type"))
setErrorResponseContext(wErr, res)
return nil, 0, wErr
}
if ok, wErr := validateMediaHeader(contentType, res.Body); !ok {
setErrorResponseContext(wErr, res)
return nil, 0, wErr
}
return res.Body, res.ContentLength, nil
}
func Upload(upload *UploadRequest) *WrappedError {
linkMeta, status, err := Post(upload.OidPath, upload.Filename)
if err != nil && status != 302 {
return Errorf(err, "Error starting file upload.")
}
oid := filepath.Base(upload.OidPath)
switch status {
case 200:
case 405, 302:
// Do the old style OPTIONS + PUT
status, err := Options(upload.OidPath)
if err != nil {
return Errorf(err, "Error getting options for file %s (%s)", upload.Filename, oid)
}
if status != 200 {
err = Put(upload.OidPath, upload.Filename, upload.CopyCallback)
if err != nil {
return Errorf(err, "Error uploading file %s (%s)", upload.Filename, oid)
}
}
case 201:
err = ExternalPut(upload.OidPath, upload.Filename, linkMeta, upload.CopyCallback)
if err != nil {
return Errorf(err, "Error uploading file %s (%s)", upload.Filename, oid)
}
default:
return Errorf(err, "Unexpected HTTP response: %d", status)
}
return nil
}
func Options(filehash string) (int, error) {
oid := filepath.Base(filehash)
_, err := os.Stat(filehash)
if err != nil {
return 0, err
}
2015-01-23 22:11:10 +00:00
tracerx.Printf("api_options: %s", oid)
req, creds, err := clientRequest("OPTIONS", oid)
if err != nil {
return 0, err
}
res, wErr := doRequest(req, creds)
if wErr != nil {
return 0, wErr
}
tracerx.Printf("api_options_status: %d", res.StatusCode)
return res.StatusCode, nil
}
func Put(filehash, filename string, cb CopyCallback) error {
2014-03-12 14:55:01 +00:00
if filename == "" {
filename = filehash
}
oid := filepath.Base(filehash)
file, err := os.Open(filehash)
2013-10-31 19:22:33 +00:00
if err != nil {
return err
}
defer file.Close()
2013-10-31 19:22:33 +00:00
stat, err := file.Stat()
2013-10-04 17:09:03 +00:00
if err != nil {
return err
}
req, creds, err := clientRequest("PUT", oid)
2013-10-04 17:09:03 +00:00
if err != nil {
return err
}
fileSize := stat.Size()
reader := &CallbackReader{
C: cb,
TotalSize: fileSize,
Reader: file,
}
bar := pb.StartNew(int(fileSize))
bar.SetUnits(pb.U_BYTES)
bar.Start()
req.Header.Set("Content-Type", gitMediaType)
req.Header.Set("Accept", gitMediaMetaType)
req.Body = ioutil.NopCloser(bar.NewProxyReader(reader))
req.ContentLength = fileSize
2013-10-04 17:09:03 +00:00
fmt.Printf("Sending %s\n", filename)
2015-01-23 22:11:10 +00:00
tracerx.Printf("api_put: %s %s", oid, filename)
res, wErr := doRequest(req, creds)
if wErr != nil {
return wErr
2013-10-04 17:09:03 +00:00
}
tracerx.Printf("api_put_status: %d", res.StatusCode)
2013-10-04 17:09:03 +00:00
return nil
}
func ExternalPut(filehash, filename string, lm *linkMeta, cb CopyCallback) error {
2015-01-23 22:11:10 +00:00
link, ok := lm.Links["upload"]
if !ok {
return Error(errors.New("No upload link provided"))
2015-01-23 22:11:10 +00:00
}
file, err := os.Open(filehash)
if err != nil {
return err
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return err
}
fileSize := stat.Size()
reader := &CallbackReader{
2015-01-23 22:11:10 +00:00
C: cb,
TotalSize: fileSize,
Reader: file,
}
req, err := http.NewRequest("PUT", link.Href, nil)
if err != nil {
return Error(err)
2015-01-23 22:11:10 +00:00
}
for h, v := range link.Header {
req.Header.Set(h, v)
}
bar := pb.StartNew(int(fileSize))
bar.SetUnits(pb.U_BYTES)
bar.Start()
req.Body = ioutil.NopCloser(bar.NewProxyReader(reader))
req.ContentLength = fileSize
tracerx.Printf("external_put: %s %s", filepath.Base(filehash), req.URL)
res, err := http.DefaultClient.Do(req)
if err != nil {
return Error(err)
2015-01-23 22:11:10 +00:00
}
tracerx.Printf("external_put_status: %d", res.StatusCode)
// Run the callback
if cb, ok := lm.Links["callback"]; ok {
oid := filepath.Base(filehash)
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return Error(err)
2015-01-23 22:11:10 +00:00
}
cbreq, err := http.NewRequest("POST", cb.Href, nil)
if err != nil {
return Error(err)
2015-01-23 22:11:10 +00:00
}
for h, v := range cb.Header {
cbreq.Header.Set(h, v)
}
d := fmt.Sprintf(`{"oid":"%s", "size":%d, "status":%d, "body":"%s"}`, oid, fileSize, res.StatusCode, string(body))
cbreq.Body = ioutil.NopCloser(bytes.NewBufferString(d))
tracerx.Printf("callback: %s %s", oid, cb.Href)
cbres, err := http.DefaultClient.Do(cbreq)
if err != nil {
return Error(err)
2015-01-23 22:11:10 +00:00
}
tracerx.Printf("callback_status: %d", cbres.StatusCode)
}
return nil
}
func Post(filehash, filename string) (*linkMeta, int, error) {
oid := filepath.Base(filehash)
req, creds, err := clientRequest("POST", "")
if err != nil {
return nil, 0, Error(err)
2015-01-23 22:11:10 +00:00
}
file, err := os.Open(filehash)
if err != nil {
return nil, 0, Error(err)
2015-01-23 22:11:10 +00:00
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return nil, 0, Error(err)
2015-01-23 22:11:10 +00:00
}
fileSize := stat.Size()
d := fmt.Sprintf(`{"oid":"%s", "size":%d}`, oid, fileSize)
req.Body = ioutil.NopCloser(bytes.NewBufferString(d))
req.Header.Set("Accept", gitMediaMetaType)
tracerx.Printf("api_post: %s %s", oid, filename)
res, wErr := doRequest(req, creds)
if wErr != nil {
return nil, 0, wErr
}
tracerx.Printf("api_post_status: %d", res.StatusCode)
if res.StatusCode == 201 {
var lm linkMeta
dec := json.NewDecoder(res.Body)
err := dec.Decode(&lm)
if err != nil {
return nil, res.StatusCode, Errorf(err, "Error decoding JSON from %s %s.", req.Method, req.URL)
2015-01-23 22:11:10 +00:00
}
return &lm, res.StatusCode, nil
}
return nil, res.StatusCode, nil
}
func validateMediaHeader(contentType string, reader io.Reader) (bool, *WrappedError) {
mediaType, params, err := mime.ParseMediaType(contentType)
if err != nil {
return false, Errorf(err, "Invalid Media Type: %s", contentType)
}
2015-01-23 22:11:10 +00:00
if mediaType == gitMediaType {
2015-01-23 22:11:10 +00:00
givenHeader, ok := params["header"]
if !ok {
return false, Error(fmt.Errorf("Missing Git Media header in %s", contentType))
2015-01-23 22:11:10 +00:00
}
2015-01-23 22:11:10 +00:00
fullGivenHeader := "--" + givenHeader + "\n"
2015-01-23 22:11:10 +00:00
header := make([]byte, len(fullGivenHeader))
_, err = io.ReadAtLeast(reader, header, len(fullGivenHeader))
if err != nil {
return false, Errorf(err, "Error reading response body.")
2015-01-23 22:11:10 +00:00
}
2015-01-23 22:11:10 +00:00
if string(header) != fullGivenHeader {
return false, Error(fmt.Errorf("Invalid header: %s expected, got %s", fullGivenHeader, header))
2015-01-23 22:11:10 +00:00
}
}
return true, nil
}
func doRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
res, err := HttpClient().Do(req)
var wErr *WrappedError
2014-08-08 17:31:33 +00:00
if err == RedirectError {
err = nil
}
2013-11-02 00:23:37 +00:00
if err == nil {
if res.StatusCode > 299 {
// An auth error should be 403. Could be 404 also.
if res.StatusCode < 405 {
execCreds(creds, "reject")
apierr := &ClientError{}
dec := json.NewDecoder(res.Body)
if err := dec.Decode(apierr); err != nil {
wErr = Errorf(err, "Error decoding JSON from response")
} else {
wErr = Errorf(apierr, "Invalid response: %d", res.StatusCode)
}
2013-11-02 00:23:37 +00:00
}
2014-08-08 17:31:33 +00:00
} else {
execCreds(creds, "approve")
2013-11-02 00:23:37 +00:00
}
} else if res.StatusCode != 302 { // hack for pre-release
wErr = Errorf(err, "Error sending HTTP request to %s", req.URL.String())
2014-08-08 17:31:33 +00:00
}
2014-08-08 17:31:33 +00:00
if wErr != nil {
if res != nil {
setErrorResponseContext(wErr, res)
} else {
setErrorRequestContext(wErr, req)
}
2013-11-02 00:23:37 +00:00
}
2014-08-08 17:31:33 +00:00
return res, wErr
}
2014-08-08 20:02:44 +00:00
var hiddenHeaders = map[string]bool{
"Authorization": true,
}
func setErrorRequestContext(err *WrappedError, req *http.Request) {
err.Set("Endpoint", Config.Endpoint())
2014-08-08 20:02:44 +00:00
err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
setErrorHeaderContext(err, "Response", req.Header)
2014-08-08 17:31:33 +00:00
}
func setErrorResponseContext(err *WrappedError, res *http.Response) {
2014-08-08 17:31:33 +00:00
err.Set("Status", res.Status)
2014-08-08 20:02:44 +00:00
setErrorHeaderContext(err, "Request", res.Header)
2014-08-08 17:31:33 +00:00
setErrorRequestContext(err, res.Request)
2013-11-02 00:23:37 +00:00
}
func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) {
2014-08-08 20:02:44 +00:00
for key, _ := range head {
contextKey := fmt.Sprintf("%s:%s", prefix, key)
if _, skip := hiddenHeaders[key]; skip {
err.Set(contextKey, "--")
} else {
err.Set(contextKey, head.Get(key))
}
}
}
func clientRequest(method, oid string) (*http.Request, Creds, error) {
u := ObjectUrl(oid)
req, err := http.NewRequest(method, u.String(), nil)
req.Header.Set("User-Agent", UserAgent)
if err == nil {
creds, err := credentials(u)
if err != nil {
2013-11-01 23:19:04 +00:00
return req, nil, err
}
token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
req.Header.Set("Authorization", auth)
2013-11-01 23:19:04 +00:00
return req, creds, nil
}
2013-11-01 23:19:04 +00:00
return req, nil, err
2013-10-31 21:33:57 +00:00
}
func ObjectUrl(oid string) *url.URL {
c := Config
u, _ := url.Parse(c.Endpoint())
u.Path = path.Join(u.Path, "objects", oid)
return u
}
type ClientError struct {
Message string `json:"message"`
RequestId string `json:"request_id,omitempty"`
}
func (e *ClientError) Error() string {
return e.Message
}