2016-12-12 00:28:47 +00:00
|
|
|
package tq
|
2016-06-08 16:03:05 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io/ioutil"
|
|
|
|
"os"
|
|
|
|
"strconv"
|
2017-01-04 22:33:31 +00:00
|
|
|
"strings"
|
2016-06-08 16:03:05 +00:00
|
|
|
|
2016-11-15 17:01:18 +00:00
|
|
|
"github.com/git-lfs/git-lfs/errors"
|
2016-12-19 18:02:13 +00:00
|
|
|
"github.com/git-lfs/git-lfs/lfsapi"
|
2016-11-15 17:01:18 +00:00
|
|
|
"github.com/git-lfs/git-lfs/progress"
|
2016-06-08 16:03:05 +00:00
|
|
|
"github.com/rubyist/tracerx"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
TusAdapterName = "tus"
|
|
|
|
TusVersion = "1.0.0"
|
|
|
|
)
|
|
|
|
|
|
|
|
// Adapter for tus.io protocol resumaable uploads
|
|
|
|
type tusUploadAdapter struct {
|
|
|
|
*adapterBase
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *tusUploadAdapter) ClearTempStorage() error {
|
|
|
|
// nothing to do, all temp state is on the server end
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-07-08 15:30:46 +00:00
|
|
|
func (a *tusUploadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
func (a *tusUploadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
|
|
|
|
}
|
|
|
|
|
2016-12-12 00:52:00 +00:00
|
|
|
func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error {
|
2016-12-14 17:57:34 +00:00
|
|
|
rel, err := t.Actions.Get("upload")
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
// return fmt.Errorf("No upload action for this object.")
|
2016-06-08 16:03:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Note not supporting the Creation extension since the batch API generates URLs
|
|
|
|
// Also not supporting Concatenation to support parallel uploads of chunks; forward only
|
|
|
|
|
|
|
|
// 1. Send HEAD request to determine upload start point
|
|
|
|
// Request must include Tus-Resumable header (version)
|
2016-12-14 17:57:34 +00:00
|
|
|
tracerx.Printf("xfer: sending tus.io HEAD request for %q", t.Oid)
|
2017-01-04 22:43:29 +00:00
|
|
|
req, err := a.newHTTPRequest("HEAD", rel)
|
2016-06-08 16:03:05 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-01-04 22:33:31 +00:00
|
|
|
|
2016-06-08 16:03:05 +00:00
|
|
|
req.Header.Set("Tus-Resumable", TusVersion)
|
2017-01-04 22:33:31 +00:00
|
|
|
|
2017-01-04 22:43:29 +00:00
|
|
|
res, err := a.doHTTP(t, req)
|
2016-06-08 16:03:05 +00:00
|
|
|
if err != nil {
|
2016-08-18 20:20:33 +00:00
|
|
|
return errors.NewRetriableError(err)
|
2016-06-08 16:03:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Response will contain Upload-Offset if supported
|
|
|
|
offHdr := res.Header.Get("Upload-Offset")
|
|
|
|
if len(offHdr) == 0 {
|
|
|
|
return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href)
|
|
|
|
}
|
|
|
|
offset, err := strconv.ParseInt(offHdr, 10, 64)
|
|
|
|
if err != nil || offset < 0 {
|
|
|
|
return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href)
|
|
|
|
}
|
|
|
|
// Upload-Offset=size means already completed (skip)
|
|
|
|
// Batch API will probably already detect this, but handle just in case
|
2016-12-14 17:57:34 +00:00
|
|
|
if offset >= t.Size {
|
|
|
|
tracerx.Printf("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Oid)
|
|
|
|
advanceCallbackProgress(cb, t, t.Size)
|
2016-06-08 16:03:05 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Open file for uploading
|
|
|
|
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
|
|
|
|
if err != nil {
|
2016-08-18 21:02:21 +00:00
|
|
|
return errors.Wrap(err, "tus upload")
|
2016-06-08 16:03:05 +00:00
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
// Upload-Offset=0 means start from scratch, but still send PATCH
|
|
|
|
if offset == 0 {
|
2016-12-14 17:57:34 +00:00
|
|
|
tracerx.Printf("xfer: tus.io uploading %q from start", t.Oid)
|
2016-06-08 16:03:05 +00:00
|
|
|
} else {
|
2016-12-14 17:57:34 +00:00
|
|
|
tracerx.Printf("xfer: tus.io resuming upload %q from %d", t.Oid, offset)
|
2016-06-08 16:03:05 +00:00
|
|
|
advanceCallbackProgress(cb, t, offset)
|
|
|
|
_, err := f.Seek(offset, os.SEEK_CUR)
|
|
|
|
if err != nil {
|
2016-08-18 21:02:21 +00:00
|
|
|
return errors.Wrap(err, "tus upload")
|
2016-06-08 16:03:05 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// 2. Send PATCH request with byte start point (even if 0) in Upload-Offset
|
|
|
|
// Response status must be 204
|
|
|
|
// Response Upload-Offset must be request Upload-Offset plus sent bytes
|
|
|
|
// Response may include Upload-Expires header in which case check not passed
|
|
|
|
|
2016-12-14 17:57:34 +00:00
|
|
|
tracerx.Printf("xfer: sending tus.io PATCH request for %q", t.Oid)
|
2017-01-04 22:43:29 +00:00
|
|
|
req, err = a.newHTTPRequest("PATCH", rel)
|
2016-06-08 16:03:05 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-01-04 22:33:31 +00:00
|
|
|
|
2016-06-08 16:03:05 +00:00
|
|
|
req.Header.Set("Tus-Resumable", TusVersion)
|
|
|
|
req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10))
|
|
|
|
req.Header.Set("Content-Type", "application/offset+octet-stream")
|
2016-12-14 17:57:34 +00:00
|
|
|
req.Header.Set("Content-Length", strconv.FormatInt(t.Size-offset, 10))
|
|
|
|
req.ContentLength = t.Size - offset
|
2016-06-08 16:03:05 +00:00
|
|
|
|
|
|
|
// Ensure progress callbacks made while uploading
|
|
|
|
// Wrap callback to give name context
|
|
|
|
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
|
|
|
|
if cb != nil {
|
|
|
|
return cb(t.Name, totalSize, readSoFar, readSinceLast)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
var reader io.Reader
|
|
|
|
reader = &progress.CallbackReader{
|
|
|
|
C: ccb,
|
2016-12-14 17:57:34 +00:00
|
|
|
TotalSize: t.Size,
|
2016-06-08 16:03:05 +00:00
|
|
|
Reader: f,
|
|
|
|
}
|
|
|
|
|
|
|
|
// Signal auth was ok on first read; this frees up other workers to start
|
|
|
|
if authOkFunc != nil {
|
|
|
|
reader = newStartCallbackReader(reader, func(*startCallbackReader) {
|
|
|
|
authOkFunc()
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
req.Body = ioutil.NopCloser(reader)
|
|
|
|
|
2017-01-04 22:43:29 +00:00
|
|
|
res, err = a.doHTTP(t, req)
|
2016-06-08 16:03:05 +00:00
|
|
|
if err != nil {
|
2016-08-18 20:20:33 +00:00
|
|
|
return errors.NewRetriableError(err)
|
2016-06-08 16:03:05 +00:00
|
|
|
}
|
2017-01-04 22:33:31 +00:00
|
|
|
|
|
|
|
a.apiClient.LogResponse("lfs.data.upload", res)
|
2016-06-08 16:03:05 +00:00
|
|
|
|
|
|
|
// A status code of 403 likely means that an authentication token for the
|
|
|
|
// upload has expired. This can be safely retried.
|
|
|
|
if res.StatusCode == 403 {
|
2016-08-18 21:24:11 +00:00
|
|
|
err = errors.New("http: received status 403")
|
2016-08-18 20:20:33 +00:00
|
|
|
return errors.NewRetriableError(err)
|
2016-06-08 16:03:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if res.StatusCode > 299 {
|
2017-01-04 22:33:31 +00:00
|
|
|
return errors.Wrapf(nil, "Invalid status for %s %s: %d",
|
|
|
|
req.Method,
|
|
|
|
strings.SplitN(req.URL.String(), "?", 2)[0],
|
|
|
|
res.StatusCode,
|
|
|
|
)
|
2016-06-08 16:03:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
|
|
|
|
2016-12-19 18:02:13 +00:00
|
|
|
cli := &lfsapi.Client{}
|
|
|
|
return verifyUpload(cli, t)
|
2016-06-08 16:03:05 +00:00
|
|
|
}
|
|
|
|
|
2016-08-09 21:43:29 +00:00
|
|
|
func configureTusAdapter(m *Manifest) {
|
2016-12-12 00:55:14 +00:00
|
|
|
m.RegisterNewAdapterFunc(TusAdapterName, Upload, func(name string, dir Direction) Adapter {
|
2016-06-08 16:03:05 +00:00
|
|
|
switch dir {
|
|
|
|
case Upload:
|
|
|
|
bu := &tusUploadAdapter{newAdapterBase(name, dir, nil)}
|
|
|
|
// self implements impl
|
|
|
|
bu.transferImpl = bu
|
|
|
|
return bu
|
|
|
|
case Download:
|
|
|
|
panic("Should never ask tus.io to download")
|
|
|
|
}
|
|
|
|
return nil
|
2016-08-09 21:43:29 +00:00
|
|
|
})
|
2016-06-08 16:03:05 +00:00
|
|
|
}
|