849ec82354
Also change signature of UploadObject to be independent of storage and work on streams like DownloadObject. This may be further refactored later
294 lines
6.5 KiB
Go
294 lines
6.5 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
|
|
"github.com/github/git-lfs/config"
|
|
"github.com/github/git-lfs/errutil"
|
|
"github.com/github/git-lfs/git"
|
|
"github.com/github/git-lfs/httputil"
|
|
"github.com/github/git-lfs/tools"
|
|
|
|
"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
|
|
)
|
|
|
|
func Batch(objects []*ObjectResource, operation string) ([]*ObjectResource, error) {
|
|
if len(objects) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
o := map[string]interface{}{"objects": objects, "operation": operation}
|
|
|
|
by, err := json.Marshal(o)
|
|
if err != nil {
|
|
return nil, errutil.Error(err)
|
|
}
|
|
|
|
req, err := NewBatchRequest(operation)
|
|
if err != nil {
|
|
return nil, errutil.Error(err)
|
|
}
|
|
|
|
req.Header.Set("Content-Type", MediaType)
|
|
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
|
|
req.ContentLength = int64(len(by))
|
|
req.Body = tools.NewReadSeekCloserWrapper(bytes.NewReader(by))
|
|
|
|
tracerx.Printf("api: batch %d files", len(objects))
|
|
|
|
res, objs, err := DoBatchRequest(req)
|
|
|
|
if err != nil {
|
|
|
|
if res == nil {
|
|
return nil, errutil.NewRetriableError(err)
|
|
}
|
|
|
|
if res.StatusCode == 0 {
|
|
return nil, errutil.NewRetriableError(err)
|
|
}
|
|
|
|
if errutil.IsAuthError(err) {
|
|
httputil.SetAuthType(req, res)
|
|
return Batch(objects, operation)
|
|
}
|
|
|
|
switch res.StatusCode {
|
|
case 404, 410:
|
|
tracerx.Printf("api: batch not implemented: %d", res.StatusCode)
|
|
return nil, errutil.NewNotImplementedError(nil)
|
|
}
|
|
|
|
tracerx.Printf("api error: %s", err)
|
|
return nil, errutil.Error(err)
|
|
}
|
|
httputil.LogTransfer("lfs.batch", res)
|
|
|
|
if res.StatusCode != 200 {
|
|
return nil, errutil.Error(fmt.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode))
|
|
}
|
|
|
|
return objs, nil
|
|
}
|
|
|
|
// Download will attempt to download the object with the given oid. The batched
|
|
// API will be used, but if the server does not implement the batch operations
|
|
// it will fall back to the legacy API.
|
|
func Download(oid string, size int64) (io.ReadCloser, int64, error) {
|
|
if !config.Config.BatchTransfer() {
|
|
return DownloadLegacy(oid)
|
|
}
|
|
|
|
objects := []*ObjectResource{
|
|
&ObjectResource{Oid: oid, Size: size},
|
|
}
|
|
|
|
objs, err := Batch(objects, "download")
|
|
if err != nil {
|
|
if errutil.IsNotImplementedError(err) {
|
|
git.Config.SetLocal("", "lfs.batch", "false")
|
|
return DownloadLegacy(oid)
|
|
}
|
|
return nil, 0, err
|
|
}
|
|
|
|
if len(objs) != 1 { // Expecting to find one object
|
|
return nil, 0, errutil.Error(fmt.Errorf("Object not found: %s", oid))
|
|
}
|
|
|
|
return DownloadObject(objs[0])
|
|
}
|
|
|
|
// DownloadLegacy attempts to download the object for the given oid using the
|
|
// legacy API.
|
|
func DownloadLegacy(oid string) (io.ReadCloser, int64, error) {
|
|
req, err := NewRequest("GET", oid)
|
|
if err != nil {
|
|
return nil, 0, errutil.Error(err)
|
|
}
|
|
|
|
res, obj, err := DoLegacyRequest(req)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
httputil.LogTransfer("lfs.download", res)
|
|
req, err = obj.NewRequest("download", "GET")
|
|
if err != nil {
|
|
return nil, 0, errutil.Error(err)
|
|
}
|
|
|
|
res, err = httputil.DoHttpRequest(req, true)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
httputil.LogTransfer("lfs.data.download", res)
|
|
|
|
return res.Body, res.ContentLength, nil
|
|
}
|
|
|
|
func DownloadCheck(oid string) (*ObjectResource, error) {
|
|
req, err := NewRequest("GET", oid)
|
|
if err != nil {
|
|
return nil, errutil.Error(err)
|
|
}
|
|
|
|
res, obj, err := DoLegacyRequest(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
httputil.LogTransfer("lfs.download", res)
|
|
|
|
_, err = obj.NewRequest("download", "GET")
|
|
if err != nil {
|
|
return nil, errutil.Error(err)
|
|
}
|
|
|
|
return obj, nil
|
|
}
|
|
|
|
func DownloadObject(obj *ObjectResource) (io.ReadCloser, int64, error) {
|
|
req, err := obj.NewRequest("download", "GET")
|
|
if err != nil {
|
|
return nil, 0, errutil.Error(err)
|
|
}
|
|
|
|
res, err := httputil.DoHttpRequest(req, true)
|
|
if err != nil {
|
|
return nil, 0, errutil.NewRetriableError(err)
|
|
}
|
|
httputil.LogTransfer("lfs.data.download", res)
|
|
|
|
return res.Body, res.ContentLength, nil
|
|
}
|
|
|
|
func UploadCheck(oidPath string) (*ObjectResource, error) {
|
|
oid := filepath.Base(oidPath)
|
|
|
|
stat, err := os.Stat(oidPath)
|
|
if err != nil {
|
|
return nil, errutil.Error(err)
|
|
}
|
|
|
|
reqObj := &ObjectResource{
|
|
Oid: oid,
|
|
Size: stat.Size(),
|
|
}
|
|
|
|
by, err := json.Marshal(reqObj)
|
|
if err != nil {
|
|
return nil, errutil.Error(err)
|
|
}
|
|
|
|
req, err := NewRequest("POST", oid)
|
|
if err != nil {
|
|
return nil, errutil.Error(err)
|
|
}
|
|
|
|
req.Header.Set("Content-Type", MediaType)
|
|
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
|
|
req.ContentLength = int64(len(by))
|
|
req.Body = tools.NewReadSeekCloserWrapper(bytes.NewReader(by))
|
|
|
|
tracerx.Printf("api: uploading (%s)", oid)
|
|
res, obj, err := DoLegacyRequest(req)
|
|
|
|
if err != nil {
|
|
if errutil.IsAuthError(err) {
|
|
httputil.SetAuthType(req, res)
|
|
return UploadCheck(oidPath)
|
|
}
|
|
|
|
return nil, errutil.NewRetriableError(err)
|
|
}
|
|
httputil.LogTransfer("lfs.upload", res)
|
|
|
|
if res.StatusCode == 200 {
|
|
return nil, nil
|
|
}
|
|
|
|
if obj.Oid == "" {
|
|
obj.Oid = oid
|
|
}
|
|
if obj.Size == 0 {
|
|
obj.Size = reqObj.Size
|
|
}
|
|
|
|
return obj, nil
|
|
}
|
|
|
|
func UploadObject(obj *ObjectResource, reader io.Reader) error {
|
|
|
|
req, err := obj.NewRequest("upload", "PUT")
|
|
if err != nil {
|
|
return errutil.Error(err)
|
|
}
|
|
|
|
if len(req.Header.Get("Content-Type")) == 0 {
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
}
|
|
|
|
if req.Header.Get("Transfer-Encoding") == "chunked" {
|
|
req.TransferEncoding = []string{"chunked"}
|
|
} else {
|
|
req.Header.Set("Content-Length", strconv.FormatInt(obj.Size, 10))
|
|
}
|
|
|
|
req.ContentLength = obj.Size
|
|
req.Body = ioutil.NopCloser(reader)
|
|
|
|
res, err := httputil.DoHttpRequest(req, true)
|
|
if err != nil {
|
|
return errutil.NewRetriableError(err)
|
|
}
|
|
httputil.LogTransfer("lfs.data.upload", res)
|
|
|
|
// A status code of 403 likely means that an authentication token for the
|
|
// upload has expired. This can be safely retried.
|
|
if res.StatusCode == 403 {
|
|
return errutil.NewRetriableError(err)
|
|
}
|
|
|
|
if res.StatusCode > 299 {
|
|
return errutil.Errorf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
|
|
}
|
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
res.Body.Close()
|
|
|
|
if _, ok := obj.Rel("verify"); !ok {
|
|
return nil
|
|
}
|
|
|
|
req, err = obj.NewRequest("verify", "POST")
|
|
if err != nil {
|
|
return errutil.Error(err)
|
|
}
|
|
|
|
by, err := json.Marshal(obj)
|
|
if err != nil {
|
|
return errutil.Error(err)
|
|
}
|
|
|
|
req.Header.Set("Content-Type", MediaType)
|
|
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
|
|
req.ContentLength = int64(len(by))
|
|
req.Body = ioutil.NopCloser(bytes.NewReader(by))
|
|
res, err = DoRequest(req, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
httputil.LogTransfer("lfs.data.verify", res)
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
res.Body.Close()
|
|
|
|
return err
|
|
}
|