2016-06-08 16:03:05 +00:00
|
|
|
package transfer
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io/ioutil"
|
|
|
|
"os"
|
|
|
|
"strconv"
|
|
|
|
|
|
|
|
"github.com/github/git-lfs/api"
|
|
|
|
"github.com/github/git-lfs/errutil"
|
|
|
|
"github.com/github/git-lfs/httputil"
|
|
|
|
"github.com/github/git-lfs/progress"
|
|
|
|
"github.com/rubyist/tracerx"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
TusAdapterName = "tus"
|
|
|
|
TusVersion = "1.0.0"
|
|
|
|
)
|
|
|
|
|
|
|
|
// Adapter for tus.io protocol resumaable uploads
|
|
|
|
type tusUploadAdapter struct {
|
|
|
|
*adapterBase
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *tusUploadAdapter) ClearTempStorage() error {
|
|
|
|
// nothing to do, all temp state is on the server end
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-07-08 15:30:46 +00:00
|
|
|
func (a *tusUploadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
func (a *tusUploadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
|
2016-06-08 16:03:05 +00:00
|
|
|
rel, ok := t.Object.Rel("upload")
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("No upload action for this object.")
|
|
|
|
}
|
|
|
|
|
|
|
|
// Note not supporting the Creation extension since the batch API generates URLs
|
|
|
|
// Also not supporting Concatenation to support parallel uploads of chunks; forward only
|
|
|
|
|
|
|
|
// 1. Send HEAD request to determine upload start point
|
|
|
|
// Request must include Tus-Resumable header (version)
|
|
|
|
tracerx.Printf("xfer: sending tus.io HEAD request for %q", t.Object.Oid)
|
|
|
|
req, err := httputil.NewHttpRequest("HEAD", rel.Href, rel.Header)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
req.Header.Set("Tus-Resumable", TusVersion)
|
2016-06-14 11:55:19 +00:00
|
|
|
res, err := httputil.DoHttpRequest(req, false)
|
2016-06-08 16:03:05 +00:00
|
|
|
if err != nil {
|
|
|
|
return errutil.NewRetriableError(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Response will contain Upload-Offset if supported
|
|
|
|
offHdr := res.Header.Get("Upload-Offset")
|
|
|
|
if len(offHdr) == 0 {
|
|
|
|
return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href)
|
|
|
|
}
|
|
|
|
offset, err := strconv.ParseInt(offHdr, 10, 64)
|
|
|
|
if err != nil || offset < 0 {
|
|
|
|
return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href)
|
|
|
|
}
|
|
|
|
// Upload-Offset=size means already completed (skip)
|
|
|
|
// Batch API will probably already detect this, but handle just in case
|
|
|
|
if offset >= t.Object.Size {
|
|
|
|
tracerx.Printf("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Object.Oid)
|
|
|
|
advanceCallbackProgress(cb, t, t.Object.Size)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Open file for uploading
|
|
|
|
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
|
|
|
|
if err != nil {
|
|
|
|
return errutil.Error(err)
|
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
// Upload-Offset=0 means start from scratch, but still send PATCH
|
|
|
|
if offset == 0 {
|
|
|
|
tracerx.Printf("xfer: tus.io uploading %q from start", t.Object.Oid)
|
|
|
|
} else {
|
|
|
|
tracerx.Printf("xfer: tus.io resuming upload %q from %d", t.Object.Oid, offset)
|
|
|
|
advanceCallbackProgress(cb, t, offset)
|
|
|
|
_, err := f.Seek(offset, os.SEEK_CUR)
|
|
|
|
if err != nil {
|
|
|
|
return errutil.Error(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// 2. Send PATCH request with byte start point (even if 0) in Upload-Offset
|
|
|
|
// Response status must be 204
|
|
|
|
// Response Upload-Offset must be request Upload-Offset plus sent bytes
|
|
|
|
// Response may include Upload-Expires header in which case check not passed
|
|
|
|
|
|
|
|
tracerx.Printf("xfer: sending tus.io PATCH request for %q", t.Object.Oid)
|
|
|
|
req, err = httputil.NewHttpRequest("PATCH", rel.Href, rel.Header)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
req.Header.Set("Tus-Resumable", TusVersion)
|
|
|
|
req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10))
|
|
|
|
req.Header.Set("Content-Type", "application/offset+octet-stream")
|
|
|
|
req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size-offset, 10))
|
|
|
|
req.ContentLength = t.Object.Size - offset
|
|
|
|
|
|
|
|
// Ensure progress callbacks made while uploading
|
|
|
|
// Wrap callback to give name context
|
|
|
|
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
|
|
|
|
if cb != nil {
|
|
|
|
return cb(t.Name, totalSize, readSoFar, readSinceLast)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
var reader io.Reader
|
|
|
|
reader = &progress.CallbackReader{
|
|
|
|
C: ccb,
|
|
|
|
TotalSize: t.Object.Size,
|
|
|
|
Reader: f,
|
|
|
|
}
|
|
|
|
|
|
|
|
// Signal auth was ok on first read; this frees up other workers to start
|
|
|
|
if authOkFunc != nil {
|
|
|
|
reader = newStartCallbackReader(reader, func(*startCallbackReader) {
|
|
|
|
authOkFunc()
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
req.Body = ioutil.NopCloser(reader)
|
|
|
|
|
2016-06-14 11:55:19 +00:00
|
|
|
res, err = httputil.DoHttpRequest(req, false)
|
2016-06-08 16:03:05 +00:00
|
|
|
if err != nil {
|
|
|
|
return errutil.NewRetriableError(err)
|
|
|
|
}
|
|
|
|
httputil.LogTransfer("lfs.data.upload", res)
|
|
|
|
|
|
|
|
// A status code of 403 likely means that an authentication token for the
|
|
|
|
// upload has expired. This can be safely retried.
|
|
|
|
if res.StatusCode == 403 {
|
|
|
|
return errutil.NewRetriableError(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if res.StatusCode > 299 {
|
|
|
|
return errutil.Errorf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
|
|
|
|
}
|
|
|
|
|
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
|
|
|
|
|
|
|
return api.VerifyUpload(t.Object)
|
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
newfunc := func(name string, dir Direction) TransferAdapter {
|
|
|
|
switch dir {
|
|
|
|
case Upload:
|
|
|
|
bu := &tusUploadAdapter{newAdapterBase(name, dir, nil)}
|
|
|
|
// self implements impl
|
|
|
|
bu.transferImpl = bu
|
|
|
|
return bu
|
|
|
|
case Download:
|
|
|
|
panic("Should never ask tus.io to download")
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
RegisterNewTransferAdapterFunc(TusAdapterName, Upload, newfunc)
|
|
|
|
}
|