2015-03-19 19:30:55 +00:00
|
|
|
package lfs
|
2013-10-04 17:09:03 +00:00
|
|
|
|
|
|
|
import (
|
2015-03-20 01:55:40 +00:00
|
|
|
"bytes"
|
2013-10-31 22:12:30 +00:00
|
|
|
"encoding/base64"
|
2015-03-19 23:20:39 +00:00
|
|
|
"encoding/json"
|
2014-03-27 14:59:32 +00:00
|
|
|
"errors"
|
2013-10-04 17:09:03 +00:00
|
|
|
"fmt"
|
2013-10-22 18:21:01 +00:00
|
|
|
"io"
|
2015-03-19 23:20:39 +00:00
|
|
|
"io/ioutil"
|
2013-10-04 17:09:03 +00:00
|
|
|
"net/http"
|
2015-03-20 01:55:40 +00:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2015-03-19 23:20:39 +00:00
|
|
|
"regexp"
|
2015-03-20 01:55:40 +00:00
|
|
|
"strconv"
|
2015-05-13 19:43:41 +00:00
|
|
|
|
2015-05-25 18:20:50 +00:00
|
|
|
"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
|
2013-10-04 17:09:03 +00:00
|
|
|
)
|
|
|
|
|
2013-12-09 15:34:29 +00:00
|
|
|
const (
|
2015-05-25 13:51:32 +00:00
|
|
|
mediaType = "application/vnd.git-lfs+json; charset=utf-8"
|
2013-12-09 15:34:29 +00:00
|
|
|
)
|
|
|
|
|
2015-04-29 17:25:39 +00:00
|
|
|
// The apiEvent* statuses (and the apiEvent channel) are used by
|
|
|
|
// UploadQueue to know when it is OK to process uploads concurrently.
|
2015-04-29 15:06:23 +00:00
|
|
|
const (
|
|
|
|
apiEventFail = iota
|
|
|
|
apiEventSuccess
|
|
|
|
)
|
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
var (
|
2015-03-20 17:10:38 +00:00
|
|
|
lfsMediaTypeRE = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
|
2015-03-22 18:32:22 +00:00
|
|
|
jsonMediaTypeRE = regexp.MustCompile(`\Aapplication/json(;|\z)`)
|
2015-03-19 21:16:52 +00:00
|
|
|
objectRelationDoesNotExist = errors.New("relation does not exist")
|
|
|
|
hiddenHeaders = map[string]bool{
|
|
|
|
"Authorization": true,
|
2015-02-12 23:22:54 +00:00
|
|
|
}
|
2015-03-19 23:20:39 +00:00
|
|
|
|
|
|
|
defaultErrors = map[int]string{
|
|
|
|
400: "Client error: %s",
|
|
|
|
401: "Authorization error: %s\nCheck that you have proper access to the repository",
|
2015-03-27 15:53:02 +00:00
|
|
|
403: "Authorization error: %s\nCheck that you have proper access to the repository",
|
2015-03-19 23:20:39 +00:00
|
|
|
404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
|
|
|
|
500: "Server error: %s",
|
|
|
|
}
|
2015-04-29 15:06:23 +00:00
|
|
|
|
|
|
|
apiEvent = make(chan int)
|
2015-03-19 21:16:52 +00:00
|
|
|
)
|
2015-02-12 02:28:42 +00:00
|
|
|
|
2015-02-26 01:09:53 +00:00
|
|
|
type objectResource struct {
|
|
|
|
Oid string `json:"oid,omitempty"`
|
2015-05-28 21:50:00 +00:00
|
|
|
Size int64 `json:"size"`
|
2015-02-26 01:09:53 +00:00
|
|
|
Links map[string]*linkRelation `json:"_links,omitempty"`
|
|
|
|
}
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) {
|
2015-02-26 01:13:56 +00:00
|
|
|
rel, ok := o.Rel(relation)
|
|
|
|
if !ok {
|
2015-03-20 01:55:40 +00:00
|
|
|
return nil, nil, objectRelationDoesNotExist
|
2015-02-26 01:13:56 +00:00
|
|
|
}
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
req, creds, err := newClientRequest(method, rel.Href)
|
2015-02-26 01:13:56 +00:00
|
|
|
if err != nil {
|
2015-03-20 01:55:40 +00:00
|
|
|
return nil, nil, err
|
2015-02-26 01:13:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
for h, v := range rel.Header {
|
|
|
|
req.Header.Set(h, v)
|
|
|
|
}
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
return req, creds, nil
|
2015-02-26 01:13:56 +00:00
|
|
|
}
|
|
|
|
|
2015-02-26 01:09:53 +00:00
|
|
|
func (o *objectResource) Rel(name string) (*linkRelation, bool) {
|
|
|
|
if o.Links == nil {
|
|
|
|
return nil, false
|
|
|
|
}
|
|
|
|
|
|
|
|
rel, ok := o.Links[name]
|
|
|
|
return rel, ok
|
|
|
|
}
|
|
|
|
|
|
|
|
type linkRelation struct {
|
|
|
|
Href string `json:"href"`
|
|
|
|
Header map[string]string `json:"header,omitempty"`
|
|
|
|
}
|
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
type ClientError struct {
|
|
|
|
Message string `json:"message"`
|
|
|
|
DocumentationUrl string `json:"documentation_url,omitempty"`
|
|
|
|
RequestId string `json:"request_id,omitempty"`
|
2014-04-01 15:17:04 +00:00
|
|
|
}
|
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
func (e *ClientError) Error() string {
|
|
|
|
msg := e.Message
|
|
|
|
if len(e.DocumentationUrl) > 0 {
|
|
|
|
msg += "\nDocs: " + e.DocumentationUrl
|
2013-10-04 17:09:03 +00:00
|
|
|
}
|
2015-03-19 21:16:52 +00:00
|
|
|
if len(e.RequestId) > 0 {
|
|
|
|
msg += "\nRequest ID: " + e.RequestId
|
2014-08-07 17:37:04 +00:00
|
|
|
}
|
2015-03-19 21:16:52 +00:00
|
|
|
return msg
|
2013-10-04 17:09:03 +00:00
|
|
|
}
|
2013-10-22 18:21:01 +00:00
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
func Download(oid string) (io.ReadCloser, int64, *WrappedError) {
|
2015-03-20 01:55:40 +00:00
|
|
|
req, creds, err := newApiRequest("GET", oid)
|
|
|
|
if err != nil {
|
|
|
|
return nil, 0, Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
res, obj, wErr := doApiRequest(req, creds)
|
|
|
|
if wErr != nil {
|
2015-05-14 16:33:29 +00:00
|
|
|
sendApiEvent(apiEventFail)
|
2015-03-20 01:55:40 +00:00
|
|
|
return nil, 0, wErr
|
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.api.download", res)
|
2015-03-20 01:55:40 +00:00
|
|
|
|
2015-05-14 16:33:29 +00:00
|
|
|
sendApiEvent(apiEventSuccess)
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
req, creds, err = obj.NewRequest("download", "GET")
|
|
|
|
if err != nil {
|
|
|
|
return nil, 0, Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
res, wErr = doHttpRequest(req, creds)
|
|
|
|
if wErr != nil {
|
|
|
|
return nil, 0, wErr
|
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.data.download", res)
|
2015-03-20 01:55:40 +00:00
|
|
|
|
|
|
|
return res.Body, res.ContentLength, nil
|
2015-03-19 21:16:52 +00:00
|
|
|
}
|
2015-01-23 22:11:10 +00:00
|
|
|
|
2015-03-26 18:46:33 +00:00
|
|
|
type byteCloser struct {
|
|
|
|
*bytes.Reader
|
|
|
|
}
|
|
|
|
|
2015-05-13 14:23:49 +00:00
|
|
|
func DownloadCheck(oid string) (*objectResource, *WrappedError) {
|
|
|
|
req, creds, err := newApiRequest("GET", oid)
|
|
|
|
if err != nil {
|
|
|
|
return nil, Error(err)
|
|
|
|
}
|
|
|
|
|
2015-06-03 21:33:00 +00:00
|
|
|
res, obj, wErr := doApiRequest(req, creds)
|
2015-05-13 14:23:49 +00:00
|
|
|
if wErr != nil {
|
|
|
|
return nil, wErr
|
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.api.download", res)
|
2015-05-13 14:23:49 +00:00
|
|
|
|
2015-05-14 16:33:29 +00:00
|
|
|
_, _, err = obj.NewRequest("download", "GET")
|
|
|
|
if err != nil {
|
|
|
|
return nil, Error(err)
|
|
|
|
}
|
|
|
|
|
2015-05-13 14:23:49 +00:00
|
|
|
return obj, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func DownloadObject(obj *objectResource) (io.ReadCloser, int64, *WrappedError) {
|
|
|
|
req, creds, err := obj.NewRequest("download", "GET")
|
|
|
|
if err != nil {
|
|
|
|
return nil, 0, Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
res, wErr := doHttpRequest(req, creds)
|
|
|
|
if wErr != nil {
|
|
|
|
return nil, 0, wErr
|
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.data.download", res)
|
2015-05-13 14:23:49 +00:00
|
|
|
|
|
|
|
return res.Body, res.ContentLength, nil
|
|
|
|
}
|
|
|
|
|
2015-03-26 18:46:33 +00:00
|
|
|
func (b *byteCloser) Close() error {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2015-06-24 18:59:11 +00:00
|
|
|
func Batch(objects []*objectResource, operation string) ([]*objectResource, *WrappedError) {
|
2015-05-18 14:57:51 +00:00
|
|
|
if len(objects) == 0 {
|
|
|
|
return nil, nil
|
2015-05-07 18:33:51 +00:00
|
|
|
}
|
|
|
|
|
2015-06-24 20:51:07 +00:00
|
|
|
br := map[string]interface{}{"objects": objects, "operation": operation}
|
2015-05-07 18:33:51 +00:00
|
|
|
|
2015-06-24 18:59:11 +00:00
|
|
|
by, err := json.Marshal(br)
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
2015-05-07 18:33:51 +00:00
|
|
|
return nil, Error(err)
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
|
|
|
|
2015-06-18 16:31:33 +00:00
|
|
|
req, creds, err := newBatchApiRequest()
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
2015-05-07 18:33:51 +00:00
|
|
|
return nil, Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("Content-Type", mediaType)
|
|
|
|
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
|
|
|
|
req.ContentLength = int64(len(by))
|
|
|
|
req.Body = &byteCloser{bytes.NewReader(by)}
|
|
|
|
|
2015-05-12 14:35:55 +00:00
|
|
|
tracerx.Printf("api: batch %d files", len(objects))
|
2015-05-07 18:33:51 +00:00
|
|
|
res, objs, wErr := doApiBatchRequest(req, creds)
|
|
|
|
if wErr != nil {
|
2015-06-02 15:19:59 +00:00
|
|
|
if res != nil {
|
|
|
|
switch res.StatusCode {
|
|
|
|
case 404, 410:
|
|
|
|
tracerx.Printf("api: batch not implemented: %d", res.StatusCode)
|
|
|
|
sendApiEvent(apiEventFail)
|
|
|
|
return nil, Error(newNotImplError())
|
|
|
|
}
|
|
|
|
}
|
2015-05-07 18:33:51 +00:00
|
|
|
sendApiEvent(apiEventFail)
|
|
|
|
return nil, wErr
|
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.api.batch", res)
|
2015-05-07 18:33:51 +00:00
|
|
|
|
|
|
|
sendApiEvent(apiEventSuccess)
|
|
|
|
if res.StatusCode != 200 {
|
|
|
|
return nil, Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
|
|
|
|
}
|
|
|
|
|
|
|
|
return objs, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func UploadCheck(oidPath string) (*objectResource, *WrappedError) {
|
|
|
|
oid := filepath.Base(oidPath)
|
|
|
|
|
2015-05-18 15:00:20 +00:00
|
|
|
stat, err := os.Stat(oidPath)
|
2015-05-07 18:33:51 +00:00
|
|
|
if err != nil {
|
|
|
|
sendApiEvent(apiEventFail)
|
|
|
|
return nil, Error(err)
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
reqObj := &objectResource{
|
|
|
|
Oid: oid,
|
|
|
|
Size: stat.Size(),
|
|
|
|
}
|
|
|
|
|
|
|
|
by, err := json.Marshal(reqObj)
|
|
|
|
if err != nil {
|
2015-05-07 18:33:51 +00:00
|
|
|
sendApiEvent(apiEventFail)
|
|
|
|
return nil, Error(err)
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
|
|
|
|
2015-03-29 23:13:50 +00:00
|
|
|
req, creds, err := newApiRequest("POST", oid)
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
2015-05-07 18:33:51 +00:00
|
|
|
sendApiEvent(apiEventFail)
|
|
|
|
return nil, Error(err)
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("Content-Type", mediaType)
|
|
|
|
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
|
2015-03-21 17:30:31 +00:00
|
|
|
req.ContentLength = int64(len(by))
|
2015-03-26 18:46:33 +00:00
|
|
|
req.Body = &byteCloser{bytes.NewReader(by)}
|
2015-03-20 01:55:40 +00:00
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
tracerx.Printf("api: uploading (%s)", oid)
|
2015-03-20 01:55:40 +00:00
|
|
|
res, obj, wErr := doApiRequest(req, creds)
|
|
|
|
if wErr != nil {
|
2015-04-29 17:25:39 +00:00
|
|
|
sendApiEvent(apiEventFail)
|
2015-05-07 18:33:51 +00:00
|
|
|
return nil, wErr
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.api.upload", res)
|
2015-03-20 01:55:40 +00:00
|
|
|
|
2015-04-29 17:25:39 +00:00
|
|
|
sendApiEvent(apiEventSuccess)
|
2015-04-29 15:06:23 +00:00
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
if res.StatusCode == 200 {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
2015-06-19 17:53:38 +00:00
|
|
|
if obj.Oid == "" {
|
|
|
|
obj.Oid = oid
|
|
|
|
}
|
|
|
|
if obj.Size == 0 {
|
|
|
|
obj.Size = reqObj.Size
|
|
|
|
}
|
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
return obj, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func UploadObject(o *objectResource, cb CopyCallback) *WrappedError {
|
|
|
|
path, err := LocalMediaPath(o.Oid)
|
|
|
|
if err != nil {
|
|
|
|
return Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
file, err := os.Open(path)
|
|
|
|
if err != nil {
|
|
|
|
return Error(err)
|
|
|
|
}
|
|
|
|
defer file.Close()
|
|
|
|
|
2015-04-30 15:43:04 +00:00
|
|
|
reader := &CallbackReader{
|
|
|
|
C: cb,
|
2015-05-07 18:33:51 +00:00
|
|
|
TotalSize: o.Size,
|
2015-04-30 15:43:04 +00:00
|
|
|
Reader: file,
|
|
|
|
}
|
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
req, creds, err := o.NewRequest("upload", "PUT")
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
|
|
|
return Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(req.Header.Get("Content-Type")) == 0 {
|
|
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
|
|
}
|
2015-05-07 18:33:51 +00:00
|
|
|
req.Header.Set("Content-Length", strconv.FormatInt(o.Size, 10))
|
|
|
|
req.ContentLength = o.Size
|
2015-03-20 18:01:43 +00:00
|
|
|
|
2015-04-24 15:43:38 +00:00
|
|
|
req.Body = ioutil.NopCloser(reader)
|
2015-03-20 01:55:40 +00:00
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
res, wErr := doHttpRequest(req, creds)
|
2015-03-20 01:55:40 +00:00
|
|
|
if wErr != nil {
|
|
|
|
return wErr
|
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.data.upload", res)
|
2015-03-20 01:55:40 +00:00
|
|
|
|
|
|
|
if res.StatusCode > 299 {
|
|
|
|
return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
|
|
|
|
}
|
|
|
|
|
2015-03-22 18:30:27 +00:00
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
req, creds, err = o.NewRequest("verify", "POST")
|
2015-03-20 01:55:40 +00:00
|
|
|
if err == objectRelationDoesNotExist {
|
|
|
|
return nil
|
|
|
|
} else if err != nil {
|
|
|
|
return Error(err)
|
|
|
|
}
|
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
by, err := json.Marshal(o)
|
|
|
|
if err != nil {
|
|
|
|
return Error(err)
|
|
|
|
}
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
req.Header.Set("Content-Type", mediaType)
|
|
|
|
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
|
2015-03-21 17:30:31 +00:00
|
|
|
req.ContentLength = int64(len(by))
|
2015-03-20 01:55:40 +00:00
|
|
|
req.Body = ioutil.NopCloser(bytes.NewReader(by))
|
2015-03-22 18:30:27 +00:00
|
|
|
res, wErr = doHttpRequest(req, creds)
|
2015-06-03 21:33:00 +00:00
|
|
|
LogTransfer("lfs.data.verify", res)
|
2015-03-22 18:30:27 +00:00
|
|
|
|
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
2015-03-20 01:55:40 +00:00
|
|
|
|
|
|
|
return wErr
|
2015-01-23 22:11:10 +00:00
|
|
|
}
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
|
2015-06-03 21:33:00 +00:00
|
|
|
res, err := DoHTTP(req)
|
2015-03-19 23:20:39 +00:00
|
|
|
|
|
|
|
var wErr *WrappedError
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL)
|
|
|
|
} else {
|
2015-06-17 20:38:09 +00:00
|
|
|
saveCredentials(creds, res)
|
2015-03-19 23:20:39 +00:00
|
|
|
wErr = handleResponse(res)
|
|
|
|
}
|
|
|
|
|
|
|
|
if wErr != nil {
|
|
|
|
if res != nil {
|
|
|
|
setErrorResponseContext(wErr, res)
|
|
|
|
} else {
|
|
|
|
setErrorRequestContext(wErr, req)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return res, wErr
|
|
|
|
}
|
|
|
|
|
2015-03-26 18:46:33 +00:00
|
|
|
func doApiRequestWithRedirects(req *http.Request, creds Creds, via []*http.Request) (*http.Response, *WrappedError) {
|
2015-03-20 01:55:40 +00:00
|
|
|
res, wErr := doHttpRequest(req, creds)
|
2015-03-26 18:46:33 +00:00
|
|
|
if wErr != nil {
|
2015-06-02 15:19:59 +00:00
|
|
|
return res, wErr
|
2015-03-26 18:46:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if res.StatusCode == 307 {
|
|
|
|
redirectedReq, redirectedCreds, err := newClientRequest(req.Method, res.Header.Get("Location"))
|
|
|
|
if err != nil {
|
2015-06-02 15:19:59 +00:00
|
|
|
return res, Errorf(err, err.Error())
|
2015-03-26 18:46:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
via = append(via, req)
|
2015-06-03 21:33:00 +00:00
|
|
|
|
|
|
|
// Avoid seeking and re-wrapping the countingReadCloser, just get the "real" body
|
|
|
|
realBody := req.Body
|
|
|
|
if wrappedBody, ok := req.Body.(*countingReadCloser); ok {
|
|
|
|
realBody = wrappedBody.ReadCloser
|
|
|
|
}
|
|
|
|
|
|
|
|
seeker, ok := realBody.(io.Seeker)
|
2015-05-11 14:42:31 +00:00
|
|
|
if !ok {
|
2015-06-02 15:19:59 +00:00
|
|
|
return res, Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.")
|
2015-03-26 18:46:33 +00:00
|
|
|
}
|
|
|
|
|
2015-05-11 14:42:31 +00:00
|
|
|
if _, err := seeker.Seek(0, 0); err != nil {
|
2015-06-02 15:19:59 +00:00
|
|
|
return res, Error(err)
|
2015-05-11 14:42:31 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
redirectedReq.Body = realBody
|
2015-05-11 14:42:31 +00:00
|
|
|
redirectedReq.ContentLength = req.ContentLength
|
|
|
|
|
2015-03-26 18:46:33 +00:00
|
|
|
if err = checkRedirect(redirectedReq, via); err != nil {
|
2015-06-02 15:19:59 +00:00
|
|
|
return res, Errorf(err, err.Error())
|
2015-03-26 18:46:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return doApiRequestWithRedirects(redirectedReq, redirectedCreds, via)
|
|
|
|
}
|
|
|
|
|
2015-05-11 14:42:23 +00:00
|
|
|
return res, nil
|
2015-03-26 18:46:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) {
|
|
|
|
via := make([]*http.Request, 0, 4)
|
|
|
|
res, wErr := doApiRequestWithRedirects(req, creds, via)
|
2015-03-20 01:55:40 +00:00
|
|
|
if wErr != nil {
|
2015-06-02 15:19:59 +00:00
|
|
|
return res, nil, wErr
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
obj := &objectResource{}
|
|
|
|
wErr = decodeApiResponse(res, obj)
|
|
|
|
|
|
|
|
if wErr != nil {
|
|
|
|
setErrorResponseContext(wErr, res)
|
2015-05-11 14:41:24 +00:00
|
|
|
return nil, nil, wErr
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
|
|
|
|
2015-05-11 14:41:24 +00:00
|
|
|
return res, obj, nil
|
2015-03-20 01:55:40 +00:00
|
|
|
}
|
|
|
|
|
2015-06-24 20:47:10 +00:00
|
|
|
// doApiBatchRequest runs the request to the batch API. If the API returns a 401,
|
2015-06-24 20:18:41 +00:00
|
|
|
// the repo will be marked as having private access and the request will be
|
|
|
|
// re-run. When the repo is marked as having private access, credentials will
|
|
|
|
// be retrieved.
|
2015-05-07 18:33:51 +00:00
|
|
|
func doApiBatchRequest(req *http.Request, creds Creds) (*http.Response, []*objectResource, *WrappedError) {
|
|
|
|
via := make([]*http.Request, 0, 4)
|
|
|
|
res, wErr := doApiRequestWithRedirects(req, creds, via)
|
2015-06-24 19:11:32 +00:00
|
|
|
|
|
|
|
if res != nil && res.StatusCode == 401 {
|
|
|
|
Config.SetPrivateAccess()
|
|
|
|
res, wErr = doApiRequestWithRedirects(req, creds, via)
|
|
|
|
}
|
|
|
|
|
2015-05-07 18:33:51 +00:00
|
|
|
if wErr != nil {
|
|
|
|
return res, nil, wErr
|
|
|
|
}
|
|
|
|
|
2015-05-18 14:57:51 +00:00
|
|
|
var objs map[string][]*objectResource
|
2015-05-07 18:33:51 +00:00
|
|
|
wErr = decodeApiResponse(res, &objs)
|
|
|
|
|
|
|
|
if wErr != nil {
|
|
|
|
setErrorResponseContext(wErr, res)
|
|
|
|
}
|
|
|
|
|
2015-05-18 14:57:51 +00:00
|
|
|
return res, objs["objects"], wErr
|
2015-05-07 18:33:51 +00:00
|
|
|
}
|
|
|
|
|
2015-03-19 23:20:39 +00:00
|
|
|
func handleResponse(res *http.Response) *WrappedError {
|
|
|
|
if res.StatusCode < 400 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
defer func() {
|
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
|
|
|
}()
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
cliErr := &ClientError{}
|
|
|
|
wErr := decodeApiResponse(res, cliErr)
|
|
|
|
if wErr == nil {
|
|
|
|
if len(cliErr.Message) == 0 {
|
|
|
|
wErr = defaultError(res)
|
|
|
|
} else {
|
|
|
|
wErr = Error(cliErr)
|
2015-03-19 23:20:39 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509
|
|
|
|
return wErr
|
|
|
|
}
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError {
|
2015-03-20 17:10:38 +00:00
|
|
|
ctype := res.Header.Get("Content-Type")
|
2015-03-22 18:32:22 +00:00
|
|
|
if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) {
|
2015-03-20 01:55:40 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
err := json.NewDecoder(res.Body).Decode(obj)
|
2015-03-22 18:30:27 +00:00
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
|
|
|
|
2015-03-20 01:55:40 +00:00
|
|
|
if err != nil {
|
|
|
|
return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2015-03-19 23:20:39 +00:00
|
|
|
func defaultError(res *http.Response) *WrappedError {
|
|
|
|
var msgFmt string
|
|
|
|
|
|
|
|
if f, ok := defaultErrors[res.StatusCode]; ok {
|
|
|
|
msgFmt = f
|
|
|
|
} else if res.StatusCode < 500 {
|
|
|
|
msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
|
|
|
|
} else {
|
|
|
|
msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
|
|
|
|
}
|
|
|
|
|
|
|
|
return Error(fmt.Errorf(msgFmt, res.Request.URL))
|
|
|
|
}
|
|
|
|
|
|
|
|
func saveCredentials(creds Creds, res *http.Response) {
|
|
|
|
if creds == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if res.StatusCode < 300 {
|
|
|
|
execCreds(creds, "approve")
|
|
|
|
} else if res.StatusCode == 401 {
|
|
|
|
execCreds(creds, "reject")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
func newApiRequest(method, oid string) (*http.Request, Creds, error) {
|
2015-03-29 23:13:50 +00:00
|
|
|
endpoint := Config.Endpoint()
|
|
|
|
objectOid := oid
|
|
|
|
operation := "download"
|
|
|
|
if method == "POST" {
|
2015-05-07 18:33:51 +00:00
|
|
|
if oid != "batch" {
|
|
|
|
objectOid = ""
|
|
|
|
operation = "upload"
|
|
|
|
}
|
2015-03-29 23:13:50 +00:00
|
|
|
}
|
|
|
|
|
2015-04-22 22:51:30 +00:00
|
|
|
res, err := sshAuthenticate(endpoint, operation, oid)
|
|
|
|
if err != nil {
|
|
|
|
tracerx.Printf("ssh: attempted with %s. Error: %s",
|
|
|
|
endpoint.SshUserAndHost, err.Error(),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(res.Href) > 0 {
|
|
|
|
endpoint.Url = res.Href
|
|
|
|
}
|
|
|
|
|
2015-03-29 23:13:50 +00:00
|
|
|
u, err := ObjectUrl(endpoint, objectOid)
|
2015-01-23 22:11:10 +00:00
|
|
|
if err != nil {
|
2015-03-19 21:16:52 +00:00
|
|
|
return nil, nil, err
|
2015-01-23 22:11:10 +00:00
|
|
|
}
|
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
req, creds, err := newClientRequest(method, u.String())
|
2015-05-11 14:42:34 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("Accept", mediaType)
|
|
|
|
if res.Header != nil {
|
|
|
|
for key, value := range res.Header {
|
|
|
|
req.Header.Set(key, value)
|
2015-03-30 06:00:34 +00:00
|
|
|
}
|
2015-01-23 22:11:10 +00:00
|
|
|
}
|
2015-05-11 14:42:34 +00:00
|
|
|
|
|
|
|
return req, creds, nil
|
2015-01-23 22:11:10 +00:00
|
|
|
}
|
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
func newClientRequest(method, rawurl string) (*http.Request, Creds, error) {
|
|
|
|
req, err := http.NewRequest(method, rawurl, nil)
|
2014-04-16 15:02:58 +00:00
|
|
|
if err != nil {
|
2015-05-11 14:42:36 +00:00
|
|
|
return nil, nil, err
|
2014-04-16 15:02:58 +00:00
|
|
|
}
|
2014-03-27 14:59:32 +00:00
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
req.Header.Set("User-Agent", UserAgent)
|
|
|
|
creds, err := getCreds(req)
|
2015-05-11 14:42:36 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return req, creds, nil
|
2013-10-22 18:21:01 +00:00
|
|
|
}
|
|
|
|
|
2015-06-18 16:31:33 +00:00
|
|
|
func newBatchApiRequest() (*http.Request, Creds, error) {
|
|
|
|
endpoint := Config.Endpoint()
|
|
|
|
|
|
|
|
res, err := sshAuthenticate(endpoint, "download", "")
|
|
|
|
if err != nil {
|
|
|
|
tracerx.Printf("ssh: attempted with %s. Error: %s",
|
|
|
|
endpoint.SshUserAndHost, err.Error(),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(res.Href) > 0 {
|
|
|
|
endpoint.Url = res.Href
|
|
|
|
}
|
|
|
|
|
|
|
|
u, err := ObjectUrl(endpoint, "batch")
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
req, creds, err := newBatchClientRequest("POST", u.String())
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("Accept", mediaType)
|
|
|
|
if res.Header != nil {
|
|
|
|
for key, value := range res.Header {
|
|
|
|
req.Header.Set(key, value)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return req, creds, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func newBatchClientRequest(method, rawurl string) (*http.Request, Creds, error) {
|
|
|
|
req, err := http.NewRequest(method, rawurl, nil)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("User-Agent", UserAgent)
|
|
|
|
|
|
|
|
// Get the creds if we're private
|
2015-06-18 16:46:27 +00:00
|
|
|
if Config.PrivateAccess() {
|
|
|
|
// The PrivateAccess() check can be pushed down and this block simplified
|
|
|
|
// once everything goes through the batch endpoint.
|
2015-06-18 16:31:33 +00:00
|
|
|
creds, err := getCreds(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
2015-06-18 16:46:27 +00:00
|
|
|
|
|
|
|
return req, creds, nil
|
|
|
|
}
|
2015-06-18 16:31:33 +00:00
|
|
|
|
|
|
|
return req, nil, nil
|
|
|
|
}
|
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
func getCreds(req *http.Request) (Creds, error) {
|
|
|
|
if len(req.Header.Get("Authorization")) > 0 {
|
|
|
|
return nil, nil
|
2013-11-02 00:23:37 +00:00
|
|
|
}
|
2013-11-02 00:31:11 +00:00
|
|
|
|
2015-03-19 21:16:52 +00:00
|
|
|
apiUrl, err := Config.ObjectUrl("")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2015-02-17 18:46:08 +00:00
|
|
|
}
|
|
|
|
|
2015-06-17 20:38:09 +00:00
|
|
|
if req.URL.Scheme != apiUrl.Scheme ||
|
|
|
|
req.URL.Host != apiUrl.Host {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
if apiUrl.User != nil {
|
|
|
|
if pass, ok := apiUrl.User.Password(); ok {
|
|
|
|
fmt.Fprintln(os.Stderr, "warning: configured Git LFS endpoint contains credentials")
|
|
|
|
setRequestAuth(req, apiUrl.User.Username(), pass)
|
|
|
|
return nil, nil
|
2015-02-17 18:46:08 +00:00
|
|
|
}
|
2015-06-17 20:38:09 +00:00
|
|
|
}
|
2015-02-17 18:46:08 +00:00
|
|
|
|
2015-06-17 20:38:09 +00:00
|
|
|
creds, err := credentials(req.URL)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2015-02-17 18:46:08 +00:00
|
|
|
}
|
|
|
|
|
2015-06-17 20:38:09 +00:00
|
|
|
setRequestAuth(req, creds["username"], creds["password"])
|
|
|
|
return creds, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func setRequestAuth(req *http.Request, user, pass string) {
|
|
|
|
token := fmt.Sprintf("%s:%s", user, pass)
|
|
|
|
auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
|
|
|
|
req.Header.Set("Authorization", auth)
|
2014-08-08 20:02:44 +00:00
|
|
|
}
|
|
|
|
|
2015-02-12 23:18:47 +00:00
|
|
|
func setErrorResponseContext(err *WrappedError, res *http.Response) {
|
2014-08-08 17:31:33 +00:00
|
|
|
err.Set("Status", res.Status)
|
2014-08-08 20:02:44 +00:00
|
|
|
setErrorHeaderContext(err, "Request", res.Header)
|
2014-08-08 17:31:33 +00:00
|
|
|
setErrorRequestContext(err, res.Request)
|
2013-11-02 00:23:37 +00:00
|
|
|
}
|
|
|
|
|
2015-05-11 14:42:39 +00:00
|
|
|
func setErrorRequestContext(err *WrappedError, req *http.Request) {
|
|
|
|
err.Set("Endpoint", Config.Endpoint().Url)
|
|
|
|
err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
|
|
|
|
setErrorHeaderContext(err, "Response", req.Header)
|
|
|
|
}
|
|
|
|
|
2015-02-12 23:18:47 +00:00
|
|
|
func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) {
|
2014-08-08 20:02:44 +00:00
|
|
|
for key, _ := range head {
|
|
|
|
contextKey := fmt.Sprintf("%s:%s", prefix, key)
|
|
|
|
if _, skip := hiddenHeaders[key]; skip {
|
|
|
|
err.Set(contextKey, "--")
|
|
|
|
} else {
|
|
|
|
err.Set(contextKey, head.Get(key))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2015-04-29 15:06:23 +00:00
|
|
|
|
2015-04-29 17:25:39 +00:00
|
|
|
func sendApiEvent(event int) {
|
2015-04-29 15:06:23 +00:00
|
|
|
select {
|
|
|
|
case apiEvent <- event:
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
}
|
2015-06-02 15:19:59 +00:00
|
|
|
|
|
|
|
type notImplError struct {
|
|
|
|
error
|
|
|
|
}
|
|
|
|
|
|
|
|
func (e notImplError) NotImplemented() bool {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
func newNotImplError() error {
|
|
|
|
return notImplError{errors.New("Not Implemented")}
|
|
|
|
}
|
|
|
|
|
|
|
|
func isNotImplError(err *WrappedError) bool {
|
|
|
|
type notimplerror interface {
|
|
|
|
NotImplemented() bool
|
|
|
|
}
|
|
|
|
if e, ok := err.Err.(notimplerror); ok {
|
|
|
|
return e.NotImplemented()
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|