2016-12-12 00:28:47 +00:00
|
|
|
package tq
|
2016-05-24 16:04:51 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"io"
|
|
|
|
"io/ioutil"
|
2018-07-20 22:37:31 +00:00
|
|
|
"net/http"
|
2016-05-24 16:04:51 +00:00
|
|
|
"os"
|
2016-05-25 10:28:24 +00:00
|
|
|
"path/filepath"
|
2016-05-24 16:04:51 +00:00
|
|
|
"strconv"
|
2017-01-04 22:27:07 +00:00
|
|
|
"strings"
|
2016-05-24 16:04:51 +00:00
|
|
|
|
2018-07-31 17:26:10 +00:00
|
|
|
"github.com/git-lfs/git-lfs/config"
|
2016-11-15 17:01:18 +00:00
|
|
|
"github.com/git-lfs/git-lfs/errors"
|
2017-01-06 20:31:45 +00:00
|
|
|
"github.com/git-lfs/git-lfs/lfsapi"
|
2017-11-22 02:00:50 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tools"
|
2016-05-24 16:04:51 +00:00
|
|
|
)
|
|
|
|
|
2016-05-25 15:41:47 +00:00
|
|
|
const (
|
2018-07-20 22:37:31 +00:00
|
|
|
BasicAdapterName = "basic"
|
|
|
|
defaultContentType = "application/octet-stream"
|
2016-05-25 15:41:47 +00:00
|
|
|
)
|
|
|
|
|
2016-06-08 08:37:25 +00:00
|
|
|
// Adapter for basic uploads (non resumable)
|
|
|
|
type basicUploadAdapter struct {
|
2016-06-03 15:32:46 +00:00
|
|
|
*adapterBase
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
2016-06-08 08:37:25 +00:00
|
|
|
func (a *basicUploadAdapter) ClearTempStorage() error {
|
2016-05-25 15:53:04 +00:00
|
|
|
// Should be empty already but also remove dir
|
|
|
|
return os.RemoveAll(a.tempDir())
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
2016-06-08 08:37:25 +00:00
|
|
|
func (a *basicUploadAdapter) tempDir() string {
|
2016-05-25 15:53:04 +00:00
|
|
|
// Must be dedicated to this adapter as deleted by ClearTempStorage
|
|
|
|
d := filepath.Join(os.TempDir(), "git-lfs-basic-temp")
|
|
|
|
if err := os.MkdirAll(d, 0755); err != nil {
|
|
|
|
return os.TempDir()
|
|
|
|
}
|
|
|
|
return d
|
2016-05-25 10:28:24 +00:00
|
|
|
}
|
|
|
|
|
2016-07-08 15:30:46 +00:00
|
|
|
func (a *basicUploadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
func (a *basicUploadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
|
|
|
|
}
|
|
|
|
|
2016-12-12 00:52:00 +00:00
|
|
|
func (a *basicUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error {
|
2017-03-06 21:33:58 +00:00
|
|
|
rel, err := t.Rel("upload")
|
2016-12-14 17:57:34 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
2017-03-06 19:36:21 +00:00
|
|
|
}
|
|
|
|
if rel == nil {
|
2017-03-06 22:02:31 +00:00
|
|
|
return errors.Errorf("No upload action for object: %s", t.Oid)
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
2017-01-04 22:43:29 +00:00
|
|
|
req, err := a.newHTTPRequest("PUT", rel)
|
2016-05-24 16:04:51 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if req.Header.Get("Transfer-Encoding") == "chunked" {
|
|
|
|
req.TransferEncoding = []string{"chunked"}
|
|
|
|
} else {
|
2016-12-14 17:57:34 +00:00
|
|
|
req.Header.Set("Content-Length", strconv.FormatInt(t.Size, 10))
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
2016-12-14 17:57:34 +00:00
|
|
|
req.ContentLength = t.Size
|
2016-05-24 16:04:51 +00:00
|
|
|
|
|
|
|
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
|
|
|
|
if err != nil {
|
2016-08-18 21:02:21 +00:00
|
|
|
return errors.Wrap(err, "basic upload")
|
2016-05-25 09:51:54 +00:00
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
2018-07-31 17:26:10 +00:00
|
|
|
if err := a.setContentTypeFor(req, f); err != nil {
|
2018-07-23 16:31:02 +00:00
|
|
|
return err
|
2018-07-20 22:37:31 +00:00
|
|
|
}
|
|
|
|
|
2016-05-25 09:51:54 +00:00
|
|
|
// Ensure progress callbacks made while uploading
|
2016-05-27 11:52:18 +00:00
|
|
|
// Wrap callback to give name context
|
|
|
|
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
|
2016-06-03 15:32:46 +00:00
|
|
|
if cb != nil {
|
|
|
|
return cb(t.Name, totalSize, readSoFar, readSinceLast)
|
2016-05-27 14:05:02 +00:00
|
|
|
}
|
|
|
|
return nil
|
2016-05-27 11:52:18 +00:00
|
|
|
}
|
2017-02-17 21:50:48 +00:00
|
|
|
|
2017-11-22 02:00:50 +00:00
|
|
|
cbr := tools.NewBodyWithCallback(f, t.Size, ccb)
|
2017-02-17 23:52:33 +00:00
|
|
|
var reader lfsapi.ReadSeekCloser = cbr
|
2016-05-25 09:51:54 +00:00
|
|
|
|
2016-06-03 15:32:46 +00:00
|
|
|
// Signal auth was ok on first read; this frees up other workers to start
|
|
|
|
if authOkFunc != nil {
|
2017-01-06 20:31:45 +00:00
|
|
|
reader = newStartCallbackReader(reader, func() error {
|
2016-06-03 15:32:46 +00:00
|
|
|
authOkFunc()
|
2017-01-06 20:31:45 +00:00
|
|
|
return nil
|
2016-05-25 10:03:47 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2017-01-06 20:31:45 +00:00
|
|
|
req.Body = reader
|
2016-05-24 16:04:51 +00:00
|
|
|
|
2017-04-27 16:07:12 +00:00
|
|
|
req = a.apiClient.LogRequest(req, "lfs.data.upload")
|
2018-09-28 20:58:25 +00:00
|
|
|
res, err := a.makeRequest(t, req)
|
2016-05-24 16:04:51 +00:00
|
|
|
if err != nil {
|
2018-07-31 18:24:17 +00:00
|
|
|
if errors.IsUnprocessableEntityError(err) {
|
|
|
|
// If we got an HTTP 422, we do _not_ want to retry the
|
|
|
|
// request later below, because it is likely that the
|
|
|
|
// implementing server does not support non-standard
|
|
|
|
// Content-Type headers.
|
|
|
|
//
|
|
|
|
// Instead, return immediately and wait for the
|
|
|
|
// *tq.TransferQueue to report an error message.
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2017-02-17 21:50:48 +00:00
|
|
|
// We're about to return a retriable error, meaning that this
|
|
|
|
// transfer will either be retried, or it will fail.
|
|
|
|
//
|
|
|
|
// Either way, let's decrement the number of bytes that we've
|
|
|
|
// read _so far_, so that the next iteration doesn't re-transfer
|
|
|
|
// those bytes, according to the progress meter.
|
2017-02-17 23:52:33 +00:00
|
|
|
if perr := cbr.ResetProgress(); perr != nil {
|
|
|
|
err = errors.Wrap(err, perr.Error())
|
|
|
|
}
|
2017-02-17 21:50:48 +00:00
|
|
|
|
2016-08-18 20:20:33 +00:00
|
|
|
return errors.NewRetriableError(err)
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
2017-01-04 22:27:07 +00:00
|
|
|
|
2016-05-24 16:04:51 +00:00
|
|
|
// A status code of 403 likely means that an authentication token for the
|
|
|
|
// upload has expired. This can be safely retried.
|
|
|
|
if res.StatusCode == 403 {
|
2016-08-18 21:24:11 +00:00
|
|
|
err = errors.New("http: received status 403")
|
2016-08-18 20:20:33 +00:00
|
|
|
return errors.NewRetriableError(err)
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if res.StatusCode > 299 {
|
2017-01-04 22:27:07 +00:00
|
|
|
return errors.Wrapf(nil, "Invalid status for %s %s: %d",
|
|
|
|
req.Method,
|
|
|
|
strings.SplitN(req.URL.String(), "?", 2)[0],
|
|
|
|
res.StatusCode,
|
|
|
|
)
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
|
|
|
|
2017-03-24 21:27:07 +00:00
|
|
|
return verifyUpload(a.apiClient, a.remote, t)
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
2018-07-31 17:26:10 +00:00
|
|
|
func (a *adapterBase) setContentTypeFor(req *http.Request, r io.ReadSeeker) error {
|
|
|
|
uc := config.NewURLConfig(a.apiClient.GitEnv())
|
|
|
|
disabled := !uc.Bool("lfs", req.URL.String(), "contenttype", true)
|
2018-08-28 14:44:42 +00:00
|
|
|
if len(req.Header.Get("Content-Type")) != 0 {
|
2018-07-31 17:06:55 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-08-28 14:44:42 +00:00
|
|
|
var contentType string
|
|
|
|
|
|
|
|
if !disabled {
|
|
|
|
buffer := make([]byte, 512)
|
|
|
|
n, err := r.Read(buffer)
|
|
|
|
if err != nil && err != io.EOF {
|
|
|
|
return errors.Wrap(err, "content type detect")
|
|
|
|
}
|
2018-07-31 17:06:55 +00:00
|
|
|
|
2018-08-28 14:44:42 +00:00
|
|
|
contentType = http.DetectContentType(buffer[:n])
|
|
|
|
if _, err := r.Seek(0, io.SeekStart); err != nil {
|
|
|
|
return errors.Wrap(err, "content type rewind")
|
|
|
|
}
|
2018-07-31 17:06:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if contentType == "" {
|
|
|
|
contentType = defaultContentType
|
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("Content-Type", contentType)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-05-25 10:03:47 +00:00
|
|
|
// startCallbackReader is a reader wrapper which calls a function as soon as the
|
|
|
|
// first Read() call is made. This callback is only made once
|
|
|
|
type startCallbackReader struct {
|
2017-01-06 20:31:45 +00:00
|
|
|
cb func() error
|
2016-05-25 10:03:47 +00:00
|
|
|
cbDone bool
|
2017-01-06 20:31:45 +00:00
|
|
|
lfsapi.ReadSeekCloser
|
2016-05-25 10:03:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (s *startCallbackReader) Read(p []byte) (n int, err error) {
|
|
|
|
if !s.cbDone && s.cb != nil {
|
2017-01-06 20:31:45 +00:00
|
|
|
if err := s.cb(); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
2016-05-25 10:03:47 +00:00
|
|
|
s.cbDone = true
|
|
|
|
}
|
2017-01-06 20:31:45 +00:00
|
|
|
return s.ReadSeekCloser.Read(p)
|
2016-05-25 10:03:47 +00:00
|
|
|
}
|
2017-01-06 20:31:45 +00:00
|
|
|
func newStartCallbackReader(r lfsapi.ReadSeekCloser, cb func() error) *startCallbackReader {
|
|
|
|
return &startCallbackReader{
|
|
|
|
ReadSeekCloser: r,
|
|
|
|
cb: cb,
|
|
|
|
}
|
2016-05-25 10:03:47 +00:00
|
|
|
}
|
|
|
|
|
2016-08-09 21:43:29 +00:00
|
|
|
func configureBasicUploadAdapter(m *Manifest) {
|
2016-12-12 00:55:14 +00:00
|
|
|
m.RegisterNewAdapterFunc(BasicAdapterName, Upload, func(name string, dir Direction) Adapter {
|
2016-06-03 15:32:46 +00:00
|
|
|
switch dir {
|
|
|
|
case Upload:
|
2017-10-25 17:51:01 +00:00
|
|
|
bu := &basicUploadAdapter{newAdapterBase(m.fs, name, dir, nil)}
|
2016-06-03 15:32:46 +00:00
|
|
|
// self implements impl
|
|
|
|
bu.transferImpl = bu
|
|
|
|
return bu
|
2016-06-08 08:37:25 +00:00
|
|
|
case Download:
|
|
|
|
panic("Should never ask this func for basic download")
|
2016-05-31 09:44:35 +00:00
|
|
|
}
|
2016-06-03 15:32:46 +00:00
|
|
|
return nil
|
2016-08-09 21:43:29 +00:00
|
|
|
})
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
2018-09-28 20:58:25 +00:00
|
|
|
|
|
|
|
func (a *basicUploadAdapter) makeRequest(t *Transfer, req *http.Request) (*http.Response, error) {
|
|
|
|
res, err := a.doHTTP(t, req)
|
|
|
|
if errors.IsAuthError(err) && len(req.Header.Get("Authorization")) == 0 {
|
|
|
|
// Construct a new body with just the raw file and no callbacks. Since
|
|
|
|
// all progress tracking happens when the net.http code copies our
|
|
|
|
// request body into a new request, we can safely make this request
|
|
|
|
// outside of the flow of the transfer adapter, and if it fails, the
|
|
|
|
// transfer progress will be rewound at the top level
|
|
|
|
f, _ := os.OpenFile(t.Path, os.O_RDONLY, 0644)
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
req.Body = tools.NewBodyWithCallback(f, t.Size, nil)
|
|
|
|
return a.makeRequest(t, req)
|
|
|
|
}
|
|
|
|
|
|
|
|
return res, err
|
|
|
|
}
|