4a5519e1fb
No longer need to negotiate support for resuming downloads with HTTP Range, client always attempts it now and falls back on re-downloading if the server doesn't support it. Since many HTTP stores like S3 support it we might as well always try.
149 lines
3.6 KiB
Go
149 lines
3.6 KiB
Go
package transfer
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
|
|
"github.com/github/git-lfs/api"
|
|
"github.com/github/git-lfs/errutil"
|
|
"github.com/github/git-lfs/httputil"
|
|
"github.com/github/git-lfs/progress"
|
|
)
|
|
|
|
const (
|
|
BasicAdapterName = "basic"
|
|
)
|
|
|
|
// Adapter for basic uploads (non resumable)
|
|
type basicUploadAdapter struct {
|
|
*adapterBase
|
|
}
|
|
|
|
func (a *basicUploadAdapter) ClearTempStorage() error {
|
|
// Should be empty already but also remove dir
|
|
return os.RemoveAll(a.tempDir())
|
|
}
|
|
|
|
func (a *basicUploadAdapter) tempDir() string {
|
|
// Must be dedicated to this adapter as deleted by ClearTempStorage
|
|
d := filepath.Join(os.TempDir(), "git-lfs-basic-temp")
|
|
if err := os.MkdirAll(d, 0755); err != nil {
|
|
return os.TempDir()
|
|
}
|
|
return d
|
|
}
|
|
|
|
func (a *basicUploadAdapter) DoTransfer(t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
|
|
rel, ok := t.Object.Rel("upload")
|
|
if !ok {
|
|
return fmt.Errorf("No upload action for this object.")
|
|
}
|
|
|
|
req, err := httputil.NewHttpRequest("PUT", rel.Href, rel.Header)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(req.Header.Get("Content-Type")) == 0 {
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
}
|
|
|
|
if req.Header.Get("Transfer-Encoding") == "chunked" {
|
|
req.TransferEncoding = []string{"chunked"}
|
|
} else {
|
|
req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size, 10))
|
|
}
|
|
|
|
req.ContentLength = t.Object.Size
|
|
|
|
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
|
|
if err != nil {
|
|
return errutil.Error(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
// Ensure progress callbacks made while uploading
|
|
// Wrap callback to give name context
|
|
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
|
|
if cb != nil {
|
|
return cb(t.Name, totalSize, readSoFar, readSinceLast)
|
|
}
|
|
return nil
|
|
}
|
|
var reader io.Reader
|
|
reader = &progress.CallbackReader{
|
|
C: ccb,
|
|
TotalSize: t.Object.Size,
|
|
Reader: f,
|
|
}
|
|
|
|
// Signal auth was ok on first read; this frees up other workers to start
|
|
if authOkFunc != nil {
|
|
reader = newStartCallbackReader(reader, func(*startCallbackReader) {
|
|
authOkFunc()
|
|
})
|
|
}
|
|
|
|
req.Body = ioutil.NopCloser(reader)
|
|
|
|
res, err := httputil.DoHttpRequest(req, true)
|
|
if err != nil {
|
|
return errutil.NewRetriableError(err)
|
|
}
|
|
httputil.LogTransfer("lfs.data.upload", res)
|
|
|
|
// A status code of 403 likely means that an authentication token for the
|
|
// upload has expired. This can be safely retried.
|
|
if res.StatusCode == 403 {
|
|
return errutil.NewRetriableError(err)
|
|
}
|
|
|
|
if res.StatusCode > 299 {
|
|
return errutil.Errorf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
|
|
}
|
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
res.Body.Close()
|
|
|
|
return api.VerifyUpload(t.Object)
|
|
}
|
|
|
|
// startCallbackReader is a reader wrapper which calls a function as soon as the
|
|
// first Read() call is made. This callback is only made once
|
|
type startCallbackReader struct {
|
|
r io.Reader
|
|
cb func(*startCallbackReader)
|
|
cbDone bool
|
|
}
|
|
|
|
func (s *startCallbackReader) Read(p []byte) (n int, err error) {
|
|
if !s.cbDone && s.cb != nil {
|
|
s.cb(s)
|
|
s.cbDone = true
|
|
}
|
|
return s.r.Read(p)
|
|
}
|
|
func newStartCallbackReader(r io.Reader, cb func(*startCallbackReader)) *startCallbackReader {
|
|
return &startCallbackReader{r, cb, false}
|
|
}
|
|
|
|
func init() {
|
|
newfunc := func(name string, dir Direction) TransferAdapter {
|
|
switch dir {
|
|
case Upload:
|
|
bu := &basicUploadAdapter{newAdapterBase(name, dir, nil)}
|
|
// self implements impl
|
|
bu.transferImpl = bu
|
|
return bu
|
|
case Download:
|
|
panic("Should never ask this func for basic download")
|
|
}
|
|
return nil
|
|
}
|
|
RegisterNewTransferAdapterFunc(BasicAdapterName, Upload, newfunc)
|
|
}
|