2015-03-19 19:30:55 +00:00
|
|
|
package lfs
|
2013-10-04 17:09:03 +00:00
|
|
|
|
|
|
|
import (
|
2015-03-20 01:55:40 +00:00
|
|
|
"bytes"
|
2013-10-31 22:12:30 +00:00
|
|
|
"encoding/base64"
|
2015-03-19 23:20:39 +00:00
|
|
|
"encoding/json"
|
2014-03-27 14:59:32 +00:00
|
|
|
"errors"
|
2013-10-04 17:09:03 +00:00
|
|
|
"fmt"
|
2013-10-22 18:21:01 +00:00
|
|
|
"io"
|
2015-03-19 23:20:39 +00:00
|
|
|
"io/ioutil"
|
2013-10-04 17:09:03 +00:00
|
|
|
"net/http"
|
2015-06-26 16:22:16 +00:00
|
|
|
"net/url"
|
2015-03-20 01:55:40 +00:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2015-03-19 23:20:39 +00:00
|
|
|
"regexp"
|
2015-03-20 01:55:40 +00:00
|
|
|
"strconv"
|
2015-05-13 19:43:41 +00:00
|
|
|
|
2015-05-25 18:20:50 +00:00
|
|
|
"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
|
2013-10-04 17:09:03 +00:00
|
|
|
)
|
|
|
|
|
2013-12-09 15:34:29 +00:00
|
|
|
const (
|
2015-05-25 13:51:32 +00:00
|
|
|
mediaType = "application/vnd.git-lfs+json; charset=utf-8"
|
2013-12-09 15:34:29 +00:00
|
|
|
)
|
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
var (
|
2015-08-23 00:40:06 +00:00
|
|
|
lfsMediaTypeRE = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
|
|
|
|
jsonMediaTypeRE = regexp.MustCompile(`\Aapplication/json(;|\z)`)
|
|
|
|
hiddenHeaders = map[string]bool{
|
2015-03-19 21:16:52 +00:00
|
|
|
"Authorization": true,
|
2015-02-12 23:22:54 +00:00
|
|
|
}
|
2015-03-19 23:20:39 +00:00
|
|
|
|
|
|
|
defaultErrors = map[int]string{
|
|
|
|
400: "Client error: %s",
|
|
|
|
401: "Authorization error: %s\nCheck that you have proper access to the repository",
|
2015-03-27 15:53:02 +00:00
|
|
|
403: "Authorization error: %s\nCheck that you have proper access to the repository",
|
2015-03-19 23:20:39 +00:00
|
|
|
404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
|
|
|
|
500: "Server error: %s",
|
|
|
|
}
|
2015-03-19 21:16:52 +00:00
|
|
|
)
|
2015-02-12 02:28:42 +00:00
|
|
|
|
2015-08-07 13:32:28 +00:00
|
|
|
type objectError struct {
|
|
|
|
Code int `json:"code"`
|
|
|
|
Message string `json:"message"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func (e *objectError) Error() string {
|
|
|
|
return fmt.Sprintf("[%d] %s", e.Code, e.Message)
|
|
|
|
}
|
|
|
|
|
2015-02-26 01:09:53 +00:00
|
|
|
type objectResource struct {
|
2015-07-31 21:20:31 +00:00
|
|
|
Oid string `json:"oid,omitempty"`
|
|
|
|
Size int64 `json:"size"`
|
|
|
|
Actions map[string]*linkRelation `json:"actions,omitempty"`
|
2015-07-31 21:30:26 +00:00
|
|
|
Links map[string]*linkRelation `json:"_links,omitempty"`
|
2015-08-07 13:32:28 +00:00
|
|
|
Error *objectError `json:"error,omitempty"`
|
2015-02-26 01:09:53 +00:00
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
func (o *objectResource) NewRequest(relation, method string) (*http.Request, error) {
|
2015-02-26 01:13:56 +00:00
|
|
|
rel, ok := o.Rel(relation)
|
|
|
|
if !ok {
|
2015-09-01 18:32:55 +00:00
|
|
|
return nil, errors.New("relation does not exist")
|
2015-02-26 01:13:56 +00:00
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
req, err := newClientRequest(method, rel.Href, rel.Header)
|
2015-02-26 01:13:56 +00:00
|
|
|
if err != nil {
|
2015-08-27 17:57:16 +00:00
|
|
|
return nil, err
|
2015-02-26 01:13:56 +00:00
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
return req, nil
|
2015-02-26 01:13:56 +00:00
|
|
|
}
|
|
|
|
|
2015-02-26 01:09:53 +00:00
|
|
|
func (o *objectResource) Rel(name string) (*linkRelation, bool) {
|
2015-08-07 13:37:03 +00:00
|
|
|
var rel *linkRelation
|
|
|
|
var ok bool
|
2015-07-31 21:30:26 +00:00
|
|
|
|
2015-08-07 13:37:03 +00:00
|
|
|
if o.Actions != nil {
|
|
|
|
rel, ok = o.Actions[name]
|
|
|
|
} else {
|
|
|
|
rel, ok = o.Links[name]
|
2015-02-26 01:09:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return rel, ok
|
|
|
|
}
|
|
|
|
|
|
|
|
type linkRelation struct {
|
|
|
|
Href string `json:"href"`
|
|
|
|
Header map[string]string `json:"header,omitempty"`
|
|
|
|
}
|
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
type ClientError struct {
|
|
|
|
Message string `json:"message"`
|
|
|
|
DocumentationUrl string `json:"documentation_url,omitempty"`
|
|
|
|
RequestId string `json:"request_id,omitempty"`
|
2014-04-01 15:17:04 +00:00
|
|
|
}
|
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
func (e *ClientError) Error() string {
|
|
|
|
msg := e.Message
|
|
|
|
if len(e.DocumentationUrl) > 0 {
|
|
|
|
msg += "\nDocs: " + e.DocumentationUrl
|
2013-10-04 17:09:03 +00:00
|
|
|
}
|
2015-03-19 21:16:52 +00:00
|
|
|
if len(e.RequestId) > 0 {
|
|
|
|
msg += "\nRequest ID: " + e.RequestId
|
2014-08-07 17:37:04 +00:00
|
|
|
}
|
2015-03-19 21:16:52 +00:00
|
|
|
return msg
|
2013-10-04 17:09:03 +00:00
|
|
|
}
|
2013-10-22 18:21:01 +00:00
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
func Download(oid string) (io.ReadCloser, int64, error) {
|
2015-08-27 17:57:16 +00:00
|
|
|
req, err := newApiRequest("GET", oid)
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, 0, Error(err)
|
|
|
|
}
|
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
res, obj, err := doLegacyApiRequest(req)
|
2015-08-21 18:31:06 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, 0, err
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.api.download", res)
|
2015-08-27 17:57:16 +00:00
|
|
|
req, err = obj.NewRequest("download", "GET")
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, 0, Error(err)
|
|
|
|
}
|
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
res, err = doStorageRequest(req)
|
2015-08-21 18:31:06 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, 0, err
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.data.download", res)
|
2015-03-20 01:55:40 +00:00
|
|
|
|
|
|
|
return res.Body, res.ContentLength, nil
|
2015-03-19 21:16:52 +00:00
|
|
|
}
|
2015-01-23 22:11:10 +00:00
|
|
|
|
2015-03-26 18:46:33 +00:00
|
|
|
type byteCloser struct {
|
|
|
|
*bytes.Reader
|
|
|
|
}
|
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
func DownloadCheck(oid string) (*objectResource, error) {
|
2015-08-27 17:57:16 +00:00
|
|
|
req, err := newApiRequest("GET", oid)
|
2015-05-13 14:23:49 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, Error(err)
|
|
|
|
}
|
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
res, obj, err := doLegacyApiRequest(req)
|
2015-08-21 18:31:06 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2015-05-13 14:23:49 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.api.download", res)
|
2015-05-13 14:23:49 +00:00
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
_, err = obj.NewRequest("download", "GET")
|
2015-05-14 16:33:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, Error(err)
|
|
|
|
}
|
|
|
|
|
2015-05-13 14:23:49 +00:00
|
|
|
return obj, nil
|
|
|
|
}
|
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
func DownloadObject(obj *objectResource) (io.ReadCloser, int64, error) {
|
2015-08-27 17:57:16 +00:00
|
|
|
req, err := obj.NewRequest("download", "GET")
|
2015-05-13 14:23:49 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, 0, Error(err)
|
|
|
|
}
|
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
res, err := doStorageRequest(req)
|
2015-08-21 18:31:06 +00:00
|
|
|
if err != nil {
|
2015-09-02 15:17:45 +00:00
|
|
|
return nil, 0, newRetriableError(err)
|
2015-05-13 14:23:49 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.data.download", res)
|
2015-05-13 14:23:49 +00:00
|
|
|
|
|
|
|
return res.Body, res.ContentLength, nil
|
|
|
|
}
|
|
|
|
|
2015-03-26 18:46:33 +00:00
|
|
|
func (b *byteCloser) Close() error {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
func Batch(objects []*objectResource, operation string) ([]*objectResource, error) {
|
2015-05-18 14:57:51 +00:00
|
|
|
if len(objects) == 0 {
|
|
|
|
return nil, nil
|
2015-05-07 18:33:51 +00:00
|
|
|
}
|
|
|
|
|
2015-06-24 20:52:31 +00:00
|
|
|
o := map[string]interface{}{"objects": objects, "operation": operation}
|
2015-05-07 18:33:51 +00:00
|
|
|
|
|
|
|
by, err := json.Marshal(o)
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
2015-05-07 18:33:51 +00:00
|
|
|
return nil, Error(err)
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
|
|
|
|
2015-09-01 22:05:54 +00:00
|
|
|
req, err := newBatchApiRequest(operation)
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
2015-05-07 18:33:51 +00:00
|
|
|
return nil, Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("Content-Type", mediaType)
|
|
|
|
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
|
|
|
|
req.ContentLength = int64(len(by))
|
|
|
|
req.Body = &byteCloser{bytes.NewReader(by)}
|
|
|
|
|
2015-05-12 14:35:55 +00:00
|
|
|
tracerx.Printf("api: batch %d files", len(objects))
|
2015-09-01 18:32:55 +00:00
|
|
|
|
|
|
|
res, objs, err := doApiBatchRequest(req)
|
2015-08-21 18:31:06 +00:00
|
|
|
if err != nil {
|
2015-07-07 15:29:33 +00:00
|
|
|
if res == nil {
|
2015-09-02 15:32:04 +00:00
|
|
|
return nil, newRetriableError(err)
|
2015-07-07 15:29:33 +00:00
|
|
|
}
|
|
|
|
|
2015-09-08 14:09:57 +00:00
|
|
|
if res.StatusCode == 0 {
|
|
|
|
return nil, newRetriableError(err)
|
|
|
|
}
|
|
|
|
|
2015-09-04 13:18:39 +00:00
|
|
|
if IsAuthError(err) {
|
2015-09-02 18:53:43 +00:00
|
|
|
Config.SetAccess("basic")
|
2015-07-07 15:29:33 +00:00
|
|
|
tracerx.Printf("api: batch not authorized, submitting with auth")
|
|
|
|
return Batch(objects, operation)
|
2015-09-04 13:18:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
switch res.StatusCode {
|
2015-07-07 15:29:33 +00:00
|
|
|
case 404, 410:
|
|
|
|
tracerx.Printf("api: batch not implemented: %d", res.StatusCode)
|
2015-08-21 15:48:52 +00:00
|
|
|
return nil, newNotImplementedError(nil)
|
2015-06-02 15:19:59 +00:00
|
|
|
}
|
2015-08-07 14:38:26 +00:00
|
|
|
|
2015-08-21 18:31:06 +00:00
|
|
|
tracerx.Printf("api error: %s", err)
|
2015-10-06 03:55:03 +00:00
|
|
|
return nil, Error(err)
|
2015-05-07 18:33:51 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.api.batch", res)
|
2015-05-07 18:33:51 +00:00
|
|
|
|
|
|
|
if res.StatusCode != 200 {
|
2015-08-18 17:10:50 +00:00
|
|
|
return nil, Error(fmt.Errorf("Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode))
|
2015-05-07 18:33:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return objs, nil
|
|
|
|
}
|
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
func UploadCheck(oidPath string) (*objectResource, error) {
|
2015-05-07 18:33:51 +00:00
|
|
|
oid := filepath.Base(oidPath)
|
|
|
|
|
2015-05-18 15:00:20 +00:00
|
|
|
stat, err := os.Stat(oidPath)
|
2015-05-07 18:33:51 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, Error(err)
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
reqObj := &objectResource{
|
|
|
|
Oid: oid,
|
|
|
|
Size: stat.Size(),
|
|
|
|
}
|
|
|
|
|
|
|
|
by, err := json.Marshal(reqObj)
|
|
|
|
if err != nil {
|
2015-05-07 18:33:51 +00:00
|
|
|
return nil, Error(err)
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
req, err := newApiRequest("POST", oid)
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
2015-05-07 18:33:51 +00:00
|
|
|
return nil, Error(err)
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("Content-Type", mediaType)
|
|
|
|
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
|
2015-03-21 17:30:31 +00:00
|
|
|
req.ContentLength = int64(len(by))
|
2015-03-26 18:46:33 +00:00
|
|
|
req.Body = &byteCloser{bytes.NewReader(by)}
|
2015-03-20 01:55:40 +00:00
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
tracerx.Printf("api: uploading (%s)", oid)
|
2015-09-01 18:32:55 +00:00
|
|
|
res, obj, err := doLegacyApiRequest(req)
|
2015-08-21 18:31:06 +00:00
|
|
|
if err != nil {
|
2015-09-04 13:18:39 +00:00
|
|
|
if IsAuthError(err) {
|
2015-09-08 14:15:06 +00:00
|
|
|
Config.SetAccess("basic")
|
2015-09-04 13:18:39 +00:00
|
|
|
tracerx.Printf("api: upload check not authorized, submitting with auth")
|
|
|
|
return UploadCheck(oidPath)
|
|
|
|
}
|
|
|
|
|
2015-09-02 15:17:45 +00:00
|
|
|
return nil, newRetriableError(err)
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.api.upload", res)
|
2015-03-20 01:55:40 +00:00
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
if res.StatusCode == 200 {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
2015-06-19 17:53:38 +00:00
|
|
|
if obj.Oid == "" {
|
|
|
|
obj.Oid = oid
|
|
|
|
}
|
|
|
|
if obj.Size == 0 {
|
|
|
|
obj.Size = reqObj.Size
|
|
|
|
}
|
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
return obj, nil
|
|
|
|
}
|
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
func UploadObject(o *objectResource, cb CopyCallback) error {
|
2015-05-07 18:33:51 +00:00
|
|
|
path, err := LocalMediaPath(o.Oid)
|
|
|
|
if err != nil {
|
|
|
|
return Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
file, err := os.Open(path)
|
|
|
|
if err != nil {
|
|
|
|
return Error(err)
|
|
|
|
}
|
|
|
|
defer file.Close()
|
|
|
|
|
2015-04-30 15:43:04 +00:00
|
|
|
reader := &CallbackReader{
|
|
|
|
C: cb,
|
2015-05-07 18:33:51 +00:00
|
|
|
TotalSize: o.Size,
|
2015-04-30 15:43:04 +00:00
|
|
|
Reader: file,
|
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
req, err := o.NewRequest("upload", "PUT")
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
|
|
|
return Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(req.Header.Get("Content-Type")) == 0 {
|
|
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
|
|
}
|
2015-09-01 18:32:55 +00:00
|
|
|
|
2015-06-12 15:19:53 +00:00
|
|
|
if req.Header.Get("Transfer-Encoding") == "chunked" {
|
|
|
|
req.TransferEncoding = []string{"chunked"}
|
|
|
|
} else {
|
|
|
|
req.Header.Set("Content-Length", strconv.FormatInt(o.Size, 10))
|
|
|
|
}
|
2015-03-20 18:01:43 +00:00
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
req.ContentLength = o.Size
|
2015-04-24 15:43:38 +00:00
|
|
|
req.Body = ioutil.NopCloser(reader)
|
2015-03-20 01:55:40 +00:00
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
res, err := doStorageRequest(req)
|
2015-08-21 18:31:06 +00:00
|
|
|
if err != nil {
|
2015-09-02 15:17:45 +00:00
|
|
|
return newRetriableError(err)
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.data.upload", res)
|
2015-03-20 01:55:40 +00:00
|
|
|
|
2015-09-03 17:02:18 +00:00
|
|
|
// A status code of 403 likely means that an authentication token for the
|
|
|
|
// upload has expired. This can be safely retried.
|
|
|
|
if res.StatusCode == 403 {
|
|
|
|
return newRetriableError(err)
|
|
|
|
}
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
if res.StatusCode > 299 {
|
|
|
|
return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
|
|
|
|
}
|
|
|
|
|
2015-03-22 18:30:27 +00:00
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
|
|
|
|
2015-08-23 00:40:06 +00:00
|
|
|
if _, ok := o.Rel("verify"); !ok {
|
2015-03-20 01:55:40 +00:00
|
|
|
return nil
|
2015-08-23 00:40:06 +00:00
|
|
|
}
|
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
req, err = o.NewRequest("verify", "POST")
|
2015-08-23 00:40:06 +00:00
|
|
|
if err != nil {
|
2015-03-20 01:55:40 +00:00
|
|
|
return Error(err)
|
|
|
|
}
|
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
by, err := json.Marshal(o)
|
|
|
|
if err != nil {
|
|
|
|
return Error(err)
|
|
|
|
}
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
req.Header.Set("Content-Type", mediaType)
|
|
|
|
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
|
2015-03-21 17:30:31 +00:00
|
|
|
req.ContentLength = int64(len(by))
|
2015-03-20 01:55:40 +00:00
|
|
|
req.Body = ioutil.NopCloser(bytes.NewReader(by))
|
2015-09-10 14:41:21 +00:00
|
|
|
res, err = doAPIRequest(req, true)
|
2015-08-21 18:31:06 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
2015-07-08 19:44:41 +00:00
|
|
|
}
|
2015-03-22 18:30:27 +00:00
|
|
|
|
2015-07-08 19:44:41 +00:00
|
|
|
LogTransfer("lfs.data.verify", res)
|
2015-03-22 18:30:27 +00:00
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
2015-03-20 01:55:40 +00:00
|
|
|
|
2015-08-21 18:31:06 +00:00
|
|
|
return err
|
2015-01-23 22:11:10 +00:00
|
|
|
}
|
|
|
|
|
2015-08-28 20:29:49 +00:00
|
|
|
// doLegacyApiRequest runs the request to the LFS legacy API.
|
2015-09-01 18:32:55 +00:00
|
|
|
func doLegacyApiRequest(req *http.Request) (*http.Response, *objectResource, error) {
|
2015-08-28 19:59:07 +00:00
|
|
|
via := make([]*http.Request, 0, 4)
|
2015-09-01 18:32:55 +00:00
|
|
|
res, err := doApiRequestWithRedirects(req, via, true)
|
|
|
|
if err != nil {
|
|
|
|
return res, nil, err
|
2015-08-28 19:59:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
obj := &objectResource{}
|
2015-09-01 18:32:55 +00:00
|
|
|
err = decodeApiResponse(res, obj)
|
2015-08-28 19:59:07 +00:00
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
if err != nil {
|
|
|
|
setErrorResponseContext(err, res)
|
|
|
|
return nil, nil, err
|
2015-08-28 19:59:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return res, obj, nil
|
|
|
|
}
|
|
|
|
|
2015-08-28 20:29:49 +00:00
|
|
|
// doApiBatchRequest runs the request to the LFS batch API. If the API returns a
|
|
|
|
// 401, the repo will be marked as having private access and the request will be
|
2015-08-28 19:59:07 +00:00
|
|
|
// re-run. When the repo is marked as having private access, credentials will
|
|
|
|
// be retrieved.
|
2015-09-01 18:32:55 +00:00
|
|
|
func doApiBatchRequest(req *http.Request) (*http.Response, []*objectResource, error) {
|
2015-09-10 14:41:21 +00:00
|
|
|
res, err := doAPIRequest(req, Config.PrivateAccess())
|
2015-08-28 19:59:07 +00:00
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
if err != nil {
|
2015-09-04 13:18:39 +00:00
|
|
|
if res.StatusCode == 401 {
|
|
|
|
return res, nil, newAuthError(err)
|
|
|
|
}
|
2015-09-01 18:32:55 +00:00
|
|
|
return res, nil, err
|
2015-08-28 19:59:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var objs map[string][]*objectResource
|
2015-09-01 18:32:55 +00:00
|
|
|
err = decodeApiResponse(res, &objs)
|
2015-08-28 19:59:07 +00:00
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
if err != nil {
|
|
|
|
setErrorResponseContext(err, res)
|
2015-08-28 19:59:07 +00:00
|
|
|
}
|
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
return res, objs["objects"], err
|
2015-08-28 19:59:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// doStorageREquest runs the request to the storage API from a link provided by
|
|
|
|
// the "actions" or "_links" properties an LFS API response.
|
2015-09-01 18:32:55 +00:00
|
|
|
func doStorageRequest(req *http.Request) (*http.Response, error) {
|
2015-08-28 20:03:40 +00:00
|
|
|
creds, err := getCreds(req)
|
|
|
|
if err != nil {
|
2015-09-01 18:49:00 +00:00
|
|
|
return nil, err
|
2015-08-28 20:03:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return doHttpRequest(req, creds)
|
2015-08-28 19:59:07 +00:00
|
|
|
}
|
|
|
|
|
2015-08-28 20:29:49 +00:00
|
|
|
// doAPIRequest runs the request to the LFS API, without parsing the response
|
|
|
|
// body. If the API returns a 401, the repo will be marked as having private
|
|
|
|
// access and the request will be re-run. When the repo is marked as having
|
|
|
|
// private access, credentials will be retrieved.
|
2015-09-10 14:41:21 +00:00
|
|
|
func doAPIRequest(req *http.Request, useCreds bool) (*http.Response, error) {
|
2015-08-28 20:29:49 +00:00
|
|
|
via := make([]*http.Request, 0, 4)
|
|
|
|
return doApiRequestWithRedirects(req, via, useCreds)
|
|
|
|
}
|
|
|
|
|
2015-08-28 19:59:07 +00:00
|
|
|
// doHttpRequest runs the given HTTP request. LFS or Storage API requests should
|
|
|
|
// use doApiBatchRequest() or doStorageRequest() instead.
|
2015-08-21 15:48:52 +00:00
|
|
|
func doHttpRequest(req *http.Request, creds Creds) (*http.Response, error) {
|
2015-07-21 20:44:40 +00:00
|
|
|
res, err := Config.HttpClient().Do(req)
|
|
|
|
if res == nil {
|
|
|
|
res = &http.Response{
|
|
|
|
StatusCode: 0,
|
|
|
|
Header: make(http.Header),
|
|
|
|
Request: req,
|
|
|
|
Body: ioutil.NopCloser(bytes.NewBufferString("")),
|
|
|
|
}
|
|
|
|
}
|
2015-03-19 23:20:39 +00:00
|
|
|
|
|
|
|
if err != nil {
|
2015-09-22 16:06:15 +00:00
|
|
|
err = Error(err)
|
2015-03-19 23:20:39 +00:00
|
|
|
} else {
|
2015-09-01 18:32:55 +00:00
|
|
|
err = handleResponse(res, creds)
|
2015-03-19 23:20:39 +00:00
|
|
|
}
|
|
|
|
|
2015-08-21 18:31:06 +00:00
|
|
|
if err != nil {
|
2015-03-19 23:20:39 +00:00
|
|
|
if res != nil {
|
2015-08-21 18:31:06 +00:00
|
|
|
setErrorResponseContext(err, res)
|
2015-03-19 23:20:39 +00:00
|
|
|
} else {
|
2015-08-21 18:31:06 +00:00
|
|
|
setErrorRequestContext(err, req)
|
2015-03-19 23:20:39 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-08-21 18:31:06 +00:00
|
|
|
return res, err
|
2015-03-19 23:20:39 +00:00
|
|
|
}
|
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
func doApiRequestWithRedirects(req *http.Request, via []*http.Request, useCreds bool) (*http.Response, error) {
|
2015-08-27 17:57:16 +00:00
|
|
|
var creds Creds
|
|
|
|
if useCreds {
|
2015-08-28 17:02:17 +00:00
|
|
|
c, err := getCredsForAPI(req)
|
2015-08-27 17:57:16 +00:00
|
|
|
if err != nil {
|
2015-09-01 18:49:00 +00:00
|
|
|
return nil, err
|
2015-08-27 17:57:16 +00:00
|
|
|
}
|
|
|
|
creds = c
|
|
|
|
}
|
|
|
|
|
2015-08-21 18:31:06 +00:00
|
|
|
res, err := doHttpRequest(req, creds)
|
|
|
|
if err != nil {
|
|
|
|
return res, err
|
2015-03-26 18:46:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if res.StatusCode == 307 {
|
2015-06-26 16:22:16 +00:00
|
|
|
redirectTo := res.Header.Get("Location")
|
|
|
|
locurl, err := url.Parse(redirectTo)
|
|
|
|
if err == nil && !locurl.IsAbs() {
|
|
|
|
locurl = req.URL.ResolveReference(locurl)
|
|
|
|
redirectTo = locurl.String()
|
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
redirectedReq, err := newClientRequest(req.Method, redirectTo, nil)
|
2015-03-26 18:46:33 +00:00
|
|
|
if err != nil {
|
2015-06-02 15:19:59 +00:00
|
|
|
return res, Errorf(err, err.Error())
|
2015-03-26 18:46:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
via = append(via, req)
|
2015-06-03 21:33:00 +00:00
|
|
|
|
|
|
|
// Avoid seeking and re-wrapping the countingReadCloser, just get the "real" body
|
|
|
|
realBody := req.Body
|
|
|
|
if wrappedBody, ok := req.Body.(*countingReadCloser); ok {
|
|
|
|
realBody = wrappedBody.ReadCloser
|
|
|
|
}
|
|
|
|
|
|
|
|
seeker, ok := realBody.(io.Seeker)
|
2015-05-11 14:42:31 +00:00
|
|
|
if !ok {
|
2015-06-02 15:19:59 +00:00
|
|
|
return res, Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.")
|
2015-03-26 18:46:33 +00:00
|
|
|
}
|
|
|
|
|
2015-05-11 14:42:31 +00:00
|
|
|
if _, err := seeker.Seek(0, 0); err != nil {
|
2015-06-02 15:19:59 +00:00
|
|
|
return res, Error(err)
|
2015-05-11 14:42:31 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
redirectedReq.Body = realBody
|
2015-05-11 14:42:31 +00:00
|
|
|
redirectedReq.ContentLength = req.ContentLength
|
|
|
|
|
2015-03-26 18:46:33 +00:00
|
|
|
if err = checkRedirect(redirectedReq, via); err != nil {
|
2015-06-02 15:19:59 +00:00
|
|
|
return res, Errorf(err, err.Error())
|
2015-03-26 18:46:33 +00:00
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
return doApiRequestWithRedirects(redirectedReq, via, useCreds)
|
2015-03-26 18:46:33 +00:00
|
|
|
}
|
|
|
|
|
2015-05-11 14:42:23 +00:00
|
|
|
return res, nil
|
2015-03-26 18:46:33 +00:00
|
|
|
}
|
|
|
|
|
2015-09-01 18:32:55 +00:00
|
|
|
func handleResponse(res *http.Response, creds Creds) error {
|
2015-08-28 19:59:07 +00:00
|
|
|
saveCredentials(creds, res)
|
2015-05-07 18:33:51 +00:00
|
|
|
|
2015-03-19 23:20:39 +00:00
|
|
|
if res.StatusCode < 400 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
defer func() {
|
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
|
|
|
}()
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
cliErr := &ClientError{}
|
2015-08-21 18:31:06 +00:00
|
|
|
err := decodeApiResponse(res, cliErr)
|
|
|
|
if err == nil {
|
2015-03-20 01:55:40 +00:00
|
|
|
if len(cliErr.Message) == 0 {
|
2015-08-21 18:31:06 +00:00
|
|
|
err = defaultError(res)
|
2015-03-20 01:55:40 +00:00
|
|
|
} else {
|
2015-08-21 18:31:06 +00:00
|
|
|
err = Error(cliErr)
|
2015-03-19 23:20:39 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-04 13:18:39 +00:00
|
|
|
if res.StatusCode == 401 {
|
|
|
|
return newAuthError(err)
|
|
|
|
}
|
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
if res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509 {
|
2015-08-21 18:31:06 +00:00
|
|
|
return newFatalError(err)
|
2015-08-21 15:48:52 +00:00
|
|
|
}
|
|
|
|
|
2015-08-21 18:31:06 +00:00
|
|
|
return err
|
2015-03-19 23:20:39 +00:00
|
|
|
}
|
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
func decodeApiResponse(res *http.Response, obj interface{}) error {
|
2015-03-20 17:10:38 +00:00
|
|
|
ctype := res.Header.Get("Content-Type")
|
2015-03-22 18:32:22 +00:00
|
|
|
if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) {
|
2015-03-20 01:55:40 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
err := json.NewDecoder(res.Body).Decode(obj)
|
2015-03-22 18:30:27 +00:00
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
|
|
|
return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
func defaultError(res *http.Response) error {
|
2015-03-19 23:20:39 +00:00
|
|
|
var msgFmt string
|
|
|
|
|
|
|
|
if f, ok := defaultErrors[res.StatusCode]; ok {
|
|
|
|
msgFmt = f
|
|
|
|
} else if res.StatusCode < 500 {
|
|
|
|
msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
|
|
|
|
} else {
|
|
|
|
msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
|
|
|
|
}
|
|
|
|
|
|
|
|
return Error(fmt.Errorf(msgFmt, res.Request.URL))
|
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
func newApiRequest(method, oid string) (*http.Request, error) {
|
2015-03-29 23:13:50 +00:00
|
|
|
endpoint := Config.Endpoint()
|
|
|
|
objectOid := oid
|
|
|
|
operation := "download"
|
|
|
|
if method == "POST" {
|
2015-05-07 18:33:51 +00:00
|
|
|
if oid != "batch" {
|
|
|
|
objectOid = ""
|
|
|
|
operation = "upload"
|
|
|
|
}
|
2015-03-29 23:13:50 +00:00
|
|
|
}
|
|
|
|
|
2015-04-22 22:51:30 +00:00
|
|
|
res, err := sshAuthenticate(endpoint, operation, oid)
|
|
|
|
if err != nil {
|
|
|
|
tracerx.Printf("ssh: attempted with %s. Error: %s",
|
|
|
|
endpoint.SshUserAndHost, err.Error(),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(res.Href) > 0 {
|
|
|
|
endpoint.Url = res.Href
|
|
|
|
}
|
|
|
|
|
2015-03-29 23:13:50 +00:00
|
|
|
u, err := ObjectUrl(endpoint, objectOid)
|
2015-01-23 22:11:10 +00:00
|
|
|
if err != nil {
|
2015-08-27 17:57:16 +00:00
|
|
|
return nil, err
|
2015-01-23 22:11:10 +00:00
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
req, err := newClientRequest(method, u.String(), res.Header)
|
2015-05-11 14:42:34 +00:00
|
|
|
if err != nil {
|
2015-08-27 17:57:16 +00:00
|
|
|
return nil, err
|
2015-05-11 14:42:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("Accept", mediaType)
|
2015-08-27 17:57:16 +00:00
|
|
|
return req, nil
|
2015-01-23 22:11:10 +00:00
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
func newClientRequest(method, rawurl string, header map[string]string) (*http.Request, error) {
|
2015-03-19 21:16:52 +00:00
|
|
|
req, err := http.NewRequest(method, rawurl, nil)
|
2014-04-16 15:02:58 +00:00
|
|
|
if err != nil {
|
2015-08-27 17:57:16 +00:00
|
|
|
return nil, err
|
2014-04-16 15:02:58 +00:00
|
|
|
}
|
2014-03-27 14:59:32 +00:00
|
|
|
|
2015-07-28 20:51:51 +00:00
|
|
|
for key, value := range header {
|
|
|
|
req.Header.Set(key, value)
|
2015-07-28 20:03:55 +00:00
|
|
|
}
|
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
req.Header.Set("User-Agent", UserAgent)
|
2015-05-11 14:42:36 +00:00
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
return req, nil
|
2013-10-22 18:21:01 +00:00
|
|
|
}
|
|
|
|
|
2015-09-01 22:05:54 +00:00
|
|
|
func newBatchApiRequest(operation string) (*http.Request, error) {
|
2015-06-18 16:31:33 +00:00
|
|
|
endpoint := Config.Endpoint()
|
|
|
|
|
2015-09-01 22:05:54 +00:00
|
|
|
res, err := sshAuthenticate(endpoint, operation, "")
|
2015-06-18 16:31:33 +00:00
|
|
|
if err != nil {
|
2015-09-01 22:05:54 +00:00
|
|
|
tracerx.Printf("ssh: %s attempted with %s. Error: %s",
|
|
|
|
operation, endpoint.SshUserAndHost, err.Error(),
|
2015-06-18 16:31:33 +00:00
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(res.Href) > 0 {
|
|
|
|
endpoint.Url = res.Href
|
|
|
|
}
|
|
|
|
|
|
|
|
u, err := ObjectUrl(endpoint, "batch")
|
|
|
|
if err != nil {
|
2015-08-27 17:57:16 +00:00
|
|
|
return nil, err
|
2015-06-18 16:31:33 +00:00
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
req, err := newBatchClientRequest("POST", u.String())
|
2015-06-18 16:31:33 +00:00
|
|
|
if err != nil {
|
2015-08-27 17:57:16 +00:00
|
|
|
return nil, err
|
2015-06-18 16:31:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("Accept", mediaType)
|
|
|
|
if res.Header != nil {
|
|
|
|
for key, value := range res.Header {
|
|
|
|
req.Header.Set(key, value)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
return req, nil
|
2015-06-18 16:31:33 +00:00
|
|
|
}
|
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
func newBatchClientRequest(method, rawurl string) (*http.Request, error) {
|
2015-06-18 16:31:33 +00:00
|
|
|
req, err := http.NewRequest(method, rawurl, nil)
|
2015-03-19 21:16:52 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2015-02-17 18:46:08 +00:00
|
|
|
}
|
|
|
|
|
2015-06-18 16:31:33 +00:00
|
|
|
req.Header.Set("User-Agent", UserAgent)
|
2015-02-17 18:46:08 +00:00
|
|
|
|
2015-08-27 17:57:16 +00:00
|
|
|
return req, nil
|
2015-06-17 20:38:09 +00:00
|
|
|
}
|
|
|
|
|
2015-08-04 18:27:09 +00:00
|
|
|
func setRequestAuthFromUrl(req *http.Request, u *url.URL) bool {
|
|
|
|
if u.User != nil {
|
|
|
|
if pass, ok := u.User.Password(); ok {
|
|
|
|
fmt.Fprintln(os.Stderr, "warning: current Git remote contains credentials")
|
|
|
|
setRequestAuth(req, u.User.Username(), pass)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2015-06-17 20:38:09 +00:00
|
|
|
func setRequestAuth(req *http.Request, user, pass string) {
|
2015-08-28 17:02:17 +00:00
|
|
|
if len(user) == 0 && len(pass) == 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2015-06-17 20:38:09 +00:00
|
|
|
token := fmt.Sprintf("%s:%s", user, pass)
|
|
|
|
auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
|
|
|
|
req.Header.Set("Authorization", auth)
|
2014-08-08 20:02:44 +00:00
|
|
|
}
|
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
func setErrorResponseContext(err error, res *http.Response) {
|
|
|
|
ErrorSetContext(err, "Status", res.Status)
|
2014-08-08 20:02:44 +00:00
|
|
|
setErrorHeaderContext(err, "Request", res.Header)
|
2014-08-08 17:31:33 +00:00
|
|
|
setErrorRequestContext(err, res.Request)
|
2013-11-02 00:23:37 +00:00
|
|
|
}
|
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
func setErrorRequestContext(err error, req *http.Request) {
|
|
|
|
ErrorSetContext(err, "Endpoint", Config.Endpoint().Url)
|
|
|
|
ErrorSetContext(err, "URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
|
2015-05-11 14:42:39 +00:00
|
|
|
setErrorHeaderContext(err, "Response", req.Header)
|
|
|
|
}
|
|
|
|
|
2015-08-21 15:48:52 +00:00
|
|
|
func setErrorHeaderContext(err error, prefix string, head http.Header) {
|
2014-08-08 20:02:44 +00:00
|
|
|
for key, _ := range head {
|
|
|
|
contextKey := fmt.Sprintf("%s:%s", prefix, key)
|
|
|
|
if _, skip := hiddenHeaders[key]; skip {
|
2015-08-21 15:48:52 +00:00
|
|
|
ErrorSetContext(err, contextKey, "--")
|
2014-08-08 20:02:44 +00:00
|
|
|
} else {
|
2015-08-21 15:48:52 +00:00
|
|
|
ErrorSetContext(err, contextKey, head.Get(key))
|
2014-08-08 20:02:44 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|