2016-12-12 00:28:47 +00:00
|
|
|
package tq
|
2016-05-24 16:04:51 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"io"
|
|
|
|
"io/ioutil"
|
2018-07-20 22:37:31 +00:00
|
|
|
"net/http"
|
2016-05-24 16:04:51 +00:00
|
|
|
"os"
|
2016-05-25 10:28:24 +00:00
|
|
|
"path/filepath"
|
2016-05-24 16:04:51 +00:00
|
|
|
"strconv"
|
2017-01-04 22:27:07 +00:00
|
|
|
"strings"
|
2016-05-24 16:04:51 +00:00
|
|
|
|
2021-09-01 19:41:10 +00:00
|
|
|
"github.com/git-lfs/git-lfs/v3/config"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/errors"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/lfsapi"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/tools"
|
2016-05-24 16:04:51 +00:00
|
|
|
)
|
|
|
|
|
2016-05-25 15:41:47 +00:00
|
|
|
const (
|
2018-07-20 22:37:31 +00:00
|
|
|
BasicAdapterName = "basic"
|
|
|
|
defaultContentType = "application/octet-stream"
|
2016-05-25 15:41:47 +00:00
|
|
|
)
|
|
|
|
|
2016-06-08 08:37:25 +00:00
|
|
|
// Adapter for basic uploads (non resumable)
|
|
|
|
type basicUploadAdapter struct {
|
2016-06-03 15:32:46 +00:00
|
|
|
*adapterBase
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
2016-06-08 08:37:25 +00:00
|
|
|
func (a *basicUploadAdapter) tempDir() string {
|
tq: drop unused ClearTempStorage adapter method
The ClearTempStorage() method of the transfer Adapter interface,
while implemented for most adapters, is never actually called,
and moreover is dangerous in some its current implementations
if it were ever to be called.
Specifically in the basic upload and download adapters and the
SSH adapter, the ClearTempStorage() method relies on an internal
tempDir() method to return the path to the temporary directory
to be cleared, which ClearTempStorage() then removes entirely.
The tempDir() methods, however, can return os.TempDir() if they
are unable to create a local temporary directory within the
Git LFS storage directory, and also do not cache the path to
the directory they first returned. So were ClearTempStorage()
ever to be called in one of these adapters, its invocation of
tempDir() might just return the system temporary directory
(e.g., /tmp on Unix), which it would then attempt to remove.
The ClearTempStorage() method was introduced in commits
f124f0585f73279195cb20ecb4aa9e149512b48a and
940a91a8df18eaf16e9a0162999f26d59fa9f5af of PR #1265, but
has never been called by any user of the transfer
adapter interface.
We therefore just remove this method and its implemetations
from all the current tranfer adapters.
2021-07-21 03:51:32 +00:00
|
|
|
// Dedicated to this adapter rather than shared with basic download.
|
2016-05-25 15:53:04 +00:00
|
|
|
d := filepath.Join(os.TempDir(), "git-lfs-basic-temp")
|
2018-12-05 16:15:52 +00:00
|
|
|
if err := tools.MkdirAll(d, a.fs); err != nil {
|
2016-05-25 15:53:04 +00:00
|
|
|
return os.TempDir()
|
|
|
|
}
|
|
|
|
return d
|
2016-05-25 10:28:24 +00:00
|
|
|
}
|
|
|
|
|
2016-07-08 15:30:46 +00:00
|
|
|
func (a *basicUploadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
func (a *basicUploadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
|
|
|
|
}
|
|
|
|
|
2016-12-12 00:52:00 +00:00
|
|
|
func (a *basicUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error {
|
2017-03-06 21:33:58 +00:00
|
|
|
rel, err := t.Rel("upload")
|
2016-12-14 17:57:34 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
2017-03-06 19:36:21 +00:00
|
|
|
}
|
|
|
|
if rel == nil {
|
2017-03-06 22:02:31 +00:00
|
|
|
return errors.Errorf("No upload action for object: %s", t.Oid)
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
2017-01-04 22:43:29 +00:00
|
|
|
req, err := a.newHTTPRequest("PUT", rel)
|
2016-05-24 16:04:51 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if req.Header.Get("Transfer-Encoding") == "chunked" {
|
|
|
|
req.TransferEncoding = []string{"chunked"}
|
|
|
|
} else {
|
2016-12-14 17:57:34 +00:00
|
|
|
req.Header.Set("Content-Length", strconv.FormatInt(t.Size, 10))
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
2016-12-14 17:57:34 +00:00
|
|
|
req.ContentLength = t.Size
|
2016-05-24 16:04:51 +00:00
|
|
|
|
|
|
|
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
|
|
|
|
if err != nil {
|
2016-08-18 21:02:21 +00:00
|
|
|
return errors.Wrap(err, "basic upload")
|
2016-05-25 09:51:54 +00:00
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
2018-07-31 17:26:10 +00:00
|
|
|
if err := a.setContentTypeFor(req, f); err != nil {
|
2018-07-23 16:31:02 +00:00
|
|
|
return err
|
2018-07-20 22:37:31 +00:00
|
|
|
}
|
|
|
|
|
2016-05-25 09:51:54 +00:00
|
|
|
// Ensure progress callbacks made while uploading
|
2016-05-27 11:52:18 +00:00
|
|
|
// Wrap callback to give name context
|
|
|
|
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
|
2016-06-03 15:32:46 +00:00
|
|
|
if cb != nil {
|
|
|
|
return cb(t.Name, totalSize, readSoFar, readSinceLast)
|
2016-05-27 14:05:02 +00:00
|
|
|
}
|
|
|
|
return nil
|
2016-05-27 11:52:18 +00:00
|
|
|
}
|
2017-02-17 21:50:48 +00:00
|
|
|
|
2020-02-25 06:18:59 +00:00
|
|
|
cbr := tools.NewFileBodyWithCallback(f, t.Size, ccb)
|
2017-02-17 23:52:33 +00:00
|
|
|
var reader lfsapi.ReadSeekCloser = cbr
|
2016-05-25 09:51:54 +00:00
|
|
|
|
2016-06-03 15:32:46 +00:00
|
|
|
// Signal auth was ok on first read; this frees up other workers to start
|
|
|
|
if authOkFunc != nil {
|
2017-01-06 20:31:45 +00:00
|
|
|
reader = newStartCallbackReader(reader, func() error {
|
2016-06-03 15:32:46 +00:00
|
|
|
authOkFunc()
|
2017-01-06 20:31:45 +00:00
|
|
|
return nil
|
2016-05-25 10:03:47 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2017-01-06 20:31:45 +00:00
|
|
|
req.Body = reader
|
2016-05-24 16:04:51 +00:00
|
|
|
|
2017-04-27 16:07:12 +00:00
|
|
|
req = a.apiClient.LogRequest(req, "lfs.data.upload")
|
2018-09-28 20:58:25 +00:00
|
|
|
res, err := a.makeRequest(t, req)
|
2016-05-24 16:04:51 +00:00
|
|
|
if err != nil {
|
2018-07-31 18:24:17 +00:00
|
|
|
if errors.IsUnprocessableEntityError(err) {
|
|
|
|
// If we got an HTTP 422, we do _not_ want to retry the
|
|
|
|
// request later below, because it is likely that the
|
|
|
|
// implementing server does not support non-standard
|
|
|
|
// Content-Type headers.
|
|
|
|
//
|
|
|
|
// Instead, return immediately and wait for the
|
|
|
|
// *tq.TransferQueue to report an error message.
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2017-02-17 21:50:48 +00:00
|
|
|
// We're about to return a retriable error, meaning that this
|
|
|
|
// transfer will either be retried, or it will fail.
|
|
|
|
//
|
|
|
|
// Either way, let's decrement the number of bytes that we've
|
|
|
|
// read _so far_, so that the next iteration doesn't re-transfer
|
|
|
|
// those bytes, according to the progress meter.
|
2017-02-17 23:52:33 +00:00
|
|
|
if perr := cbr.ResetProgress(); perr != nil {
|
|
|
|
err = errors.Wrap(err, perr.Error())
|
|
|
|
}
|
2017-02-17 21:50:48 +00:00
|
|
|
|
2019-02-19 15:11:24 +00:00
|
|
|
if res == nil {
|
|
|
|
// We encountered a network or similar error which caused us
|
|
|
|
// to not receive a response at all.
|
|
|
|
return errors.NewRetriableError(err)
|
|
|
|
}
|
|
|
|
|
2019-01-07 09:27:02 +00:00
|
|
|
if res.StatusCode == 429 {
|
|
|
|
retLaterErr := errors.NewRetriableLaterError(err, res.Header["Retry-After"][0])
|
|
|
|
if retLaterErr != nil {
|
|
|
|
return retLaterErr
|
|
|
|
}
|
|
|
|
}
|
2016-08-18 20:20:33 +00:00
|
|
|
return errors.NewRetriableError(err)
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
2017-01-04 22:27:07 +00:00
|
|
|
|
2016-05-24 16:04:51 +00:00
|
|
|
// A status code of 403 likely means that an authentication token for the
|
|
|
|
// upload has expired. This can be safely retried.
|
|
|
|
if res.StatusCode == 403 {
|
2016-08-18 21:24:11 +00:00
|
|
|
err = errors.New("http: received status 403")
|
2016-08-18 20:20:33 +00:00
|
|
|
return errors.NewRetriableError(err)
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if res.StatusCode > 299 {
|
2017-01-04 22:27:07 +00:00
|
|
|
return errors.Wrapf(nil, "Invalid status for %s %s: %d",
|
|
|
|
req.Method,
|
|
|
|
strings.SplitN(req.URL.String(), "?", 2)[0],
|
|
|
|
res.StatusCode,
|
|
|
|
)
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
io.Copy(ioutil.Discard, res.Body)
|
|
|
|
res.Body.Close()
|
|
|
|
|
2017-03-24 21:27:07 +00:00
|
|
|
return verifyUpload(a.apiClient, a.remote, t)
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
|
|
|
|
2018-07-31 17:26:10 +00:00
|
|
|
func (a *adapterBase) setContentTypeFor(req *http.Request, r io.ReadSeeker) error {
|
|
|
|
uc := config.NewURLConfig(a.apiClient.GitEnv())
|
|
|
|
disabled := !uc.Bool("lfs", req.URL.String(), "contenttype", true)
|
2018-08-28 14:44:42 +00:00
|
|
|
if len(req.Header.Get("Content-Type")) != 0 {
|
2018-07-31 17:06:55 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-08-28 14:44:42 +00:00
|
|
|
var contentType string
|
|
|
|
|
|
|
|
if !disabled {
|
|
|
|
buffer := make([]byte, 512)
|
|
|
|
n, err := r.Read(buffer)
|
|
|
|
if err != nil && err != io.EOF {
|
|
|
|
return errors.Wrap(err, "content type detect")
|
|
|
|
}
|
2018-07-31 17:06:55 +00:00
|
|
|
|
2018-08-28 14:44:42 +00:00
|
|
|
contentType = http.DetectContentType(buffer[:n])
|
|
|
|
if _, err := r.Seek(0, io.SeekStart); err != nil {
|
|
|
|
return errors.Wrap(err, "content type rewind")
|
|
|
|
}
|
2018-07-31 17:06:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if contentType == "" {
|
|
|
|
contentType = defaultContentType
|
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("Content-Type", contentType)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-05-25 10:03:47 +00:00
|
|
|
// startCallbackReader is a reader wrapper which calls a function as soon as the
|
|
|
|
// first Read() call is made. This callback is only made once
|
|
|
|
type startCallbackReader struct {
|
2017-01-06 20:31:45 +00:00
|
|
|
cb func() error
|
2016-05-25 10:03:47 +00:00
|
|
|
cbDone bool
|
2017-01-06 20:31:45 +00:00
|
|
|
lfsapi.ReadSeekCloser
|
2016-05-25 10:03:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (s *startCallbackReader) Read(p []byte) (n int, err error) {
|
|
|
|
if !s.cbDone && s.cb != nil {
|
2017-01-06 20:31:45 +00:00
|
|
|
if err := s.cb(); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
2016-05-25 10:03:47 +00:00
|
|
|
s.cbDone = true
|
|
|
|
}
|
2017-01-06 20:31:45 +00:00
|
|
|
return s.ReadSeekCloser.Read(p)
|
2016-05-25 10:03:47 +00:00
|
|
|
}
|
2017-01-06 20:31:45 +00:00
|
|
|
func newStartCallbackReader(r lfsapi.ReadSeekCloser, cb func() error) *startCallbackReader {
|
|
|
|
return &startCallbackReader{
|
|
|
|
ReadSeekCloser: r,
|
|
|
|
cb: cb,
|
|
|
|
}
|
2016-05-25 10:03:47 +00:00
|
|
|
}
|
|
|
|
|
2016-08-09 21:43:29 +00:00
|
|
|
func configureBasicUploadAdapter(m *Manifest) {
|
2016-12-12 00:55:14 +00:00
|
|
|
m.RegisterNewAdapterFunc(BasicAdapterName, Upload, func(name string, dir Direction) Adapter {
|
2016-06-03 15:32:46 +00:00
|
|
|
switch dir {
|
|
|
|
case Upload:
|
2017-10-25 17:51:01 +00:00
|
|
|
bu := &basicUploadAdapter{newAdapterBase(m.fs, name, dir, nil)}
|
2016-06-03 15:32:46 +00:00
|
|
|
// self implements impl
|
|
|
|
bu.transferImpl = bu
|
|
|
|
return bu
|
2016-06-08 08:37:25 +00:00
|
|
|
case Download:
|
|
|
|
panic("Should never ask this func for basic download")
|
2016-05-31 09:44:35 +00:00
|
|
|
}
|
2016-06-03 15:32:46 +00:00
|
|
|
return nil
|
2016-08-09 21:43:29 +00:00
|
|
|
})
|
2016-05-24 16:04:51 +00:00
|
|
|
}
|
2018-09-28 20:58:25 +00:00
|
|
|
|
|
|
|
func (a *basicUploadAdapter) makeRequest(t *Transfer, req *http.Request) (*http.Response, error) {
|
|
|
|
res, err := a.doHTTP(t, req)
|
|
|
|
if errors.IsAuthError(err) && len(req.Header.Get("Authorization")) == 0 {
|
|
|
|
// Construct a new body with just the raw file and no callbacks. Since
|
|
|
|
// all progress tracking happens when the net.http code copies our
|
|
|
|
// request body into a new request, we can safely make this request
|
|
|
|
// outside of the flow of the transfer adapter, and if it fails, the
|
|
|
|
// transfer progress will be rewound at the top level
|
|
|
|
f, _ := os.OpenFile(t.Path, os.O_RDONLY, 0644)
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
req.Body = tools.NewBodyWithCallback(f, t.Size, nil)
|
|
|
|
return a.makeRequest(t, req)
|
|
|
|
}
|
|
|
|
|
|
|
|
return res, err
|
|
|
|
}
|