package transfer import ( "fmt" "io" "io/ioutil" "os" "strconv" "github.com/github/git-lfs/api" "github.com/github/git-lfs/errutil" "github.com/github/git-lfs/httputil" "github.com/github/git-lfs/progress" "github.com/rubyist/tracerx" ) const ( TusAdapterName = "tus" TusVersion = "1.0.0" ) // Adapter for tus.io protocol resumaable uploads type tusUploadAdapter struct { *adapterBase } func (a *tusUploadAdapter) ClearTempStorage() error { // nothing to do, all temp state is on the server end return nil } func (a *tusUploadAdapter) DoTransfer(t *Transfer, cb TransferProgressCallback, authOkFunc func()) error { rel, ok := t.Object.Rel("upload") if !ok { return fmt.Errorf("No upload action for this object.") } // Note not supporting the Creation extension since the batch API generates URLs // Also not supporting Concatenation to support parallel uploads of chunks; forward only // 1. Send HEAD request to determine upload start point // Request must include Tus-Resumable header (version) tracerx.Printf("xfer: sending tus.io HEAD request for %q", t.Object.Oid) req, err := httputil.NewHttpRequest("HEAD", rel.Href, rel.Header) if err != nil { return err } req.Header.Set("Tus-Resumable", TusVersion) res, err := httputil.DoHttpRequest(req, false) if err != nil { return errutil.NewRetriableError(err) } // Response will contain Upload-Offset if supported offHdr := res.Header.Get("Upload-Offset") if len(offHdr) == 0 { return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href) } offset, err := strconv.ParseInt(offHdr, 10, 64) if err != nil || offset < 0 { return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href) } // Upload-Offset=size means already completed (skip) // Batch API will probably already detect this, but handle just in case if offset >= t.Object.Size { tracerx.Printf("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Object.Oid) advanceCallbackProgress(cb, t, t.Object.Size) return nil } // Open file for uploading f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644) if err != nil { return errutil.Error(err) } defer f.Close() // Upload-Offset=0 means start from scratch, but still send PATCH if offset == 0 { tracerx.Printf("xfer: tus.io uploading %q from start", t.Object.Oid) } else { tracerx.Printf("xfer: tus.io resuming upload %q from %d", t.Object.Oid, offset) advanceCallbackProgress(cb, t, offset) _, err := f.Seek(offset, os.SEEK_CUR) if err != nil { return errutil.Error(err) } } // 2. Send PATCH request with byte start point (even if 0) in Upload-Offset // Response status must be 204 // Response Upload-Offset must be request Upload-Offset plus sent bytes // Response may include Upload-Expires header in which case check not passed tracerx.Printf("xfer: sending tus.io PATCH request for %q", t.Object.Oid) req, err = httputil.NewHttpRequest("PATCH", rel.Href, rel.Header) if err != nil { return err } req.Header.Set("Tus-Resumable", TusVersion) req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10)) req.Header.Set("Content-Type", "application/offset+octet-stream") req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size-offset, 10)) req.ContentLength = t.Object.Size - offset // Ensure progress callbacks made while uploading // Wrap callback to give name context ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error { if cb != nil { return cb(t.Name, totalSize, readSoFar, readSinceLast) } return nil } var reader io.Reader reader = &progress.CallbackReader{ C: ccb, TotalSize: t.Object.Size, Reader: f, } // Signal auth was ok on first read; this frees up other workers to start if authOkFunc != nil { reader = newStartCallbackReader(reader, func(*startCallbackReader) { authOkFunc() }) } req.Body = ioutil.NopCloser(reader) res, err = httputil.DoHttpRequest(req, false) if err != nil { return errutil.NewRetriableError(err) } httputil.LogTransfer("lfs.data.upload", res) // A status code of 403 likely means that an authentication token for the // upload has expired. This can be safely retried. if res.StatusCode == 403 { return errutil.NewRetriableError(err) } if res.StatusCode > 299 { return errutil.Errorf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode) } io.Copy(ioutil.Discard, res.Body) res.Body.Close() return api.VerifyUpload(t.Object) } func init() { newfunc := func(name string, dir Direction) TransferAdapter { switch dir { case Upload: bu := &tusUploadAdapter{newAdapterBase(name, dir, nil)} // self implements impl bu.transferImpl = bu return bu case Download: panic("Should never ask tus.io to download") } return nil } RegisterNewTransferAdapterFunc(TusAdapterName, Upload, newfunc) }