2015-04-23 16:20:36 +00:00
|
|
|
package lfs
|
2013-10-04 15:22:32 +00:00
|
|
|
|
|
|
|
import (
|
2015-03-22 16:01:26 +00:00
|
|
|
"fmt"
|
2013-10-04 15:22:32 +00:00
|
|
|
"io"
|
2013-11-05 15:26:58 +00:00
|
|
|
"os"
|
2015-03-20 17:30:24 +00:00
|
|
|
"path/filepath"
|
2015-05-13 19:43:41 +00:00
|
|
|
|
2017-11-22 02:00:50 +00:00
|
|
|
"github.com/git-lfs/git-lfs/config"
|
|
|
|
"github.com/git-lfs/git-lfs/errors"
|
2016-11-15 17:01:18 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tools"
|
2017-07-03 20:40:43 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tools/humanize"
|
2016-12-12 00:28:47 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tq"
|
2016-05-23 18:02:27 +00:00
|
|
|
"github.com/rubyist/tracerx"
|
2013-10-04 15:22:32 +00:00
|
|
|
)
|
|
|
|
|
2017-11-22 02:00:50 +00:00
|
|
|
func (f *GitFilter) SmudgeToFile(filename string, ptr *Pointer, download bool, manifest *tq.Manifest, cb tools.CopyCallback) error {
|
2018-12-05 16:15:52 +00:00
|
|
|
tools.MkdirAll(filepath.Dir(filename), f.cfg)
|
2018-07-09 19:15:03 +00:00
|
|
|
|
|
|
|
if stat, _ := os.Stat(filename); stat != nil && stat.Mode()&0200 == 0 {
|
|
|
|
if err := os.Chmod(filename, stat.Mode()|0200); err != nil {
|
|
|
|
return errors.Wrap(err,
|
|
|
|
"Could not restore write permission")
|
|
|
|
}
|
|
|
|
|
|
|
|
// When we're done, return the file back to its normal
|
|
|
|
// permission bits.
|
|
|
|
defer os.Chmod(filename, stat.Mode())
|
|
|
|
}
|
|
|
|
|
2019-08-16 15:12:36 +00:00
|
|
|
abs, err := filepath.Abs(filename)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Could not produce absolute path for %q", filename)
|
|
|
|
}
|
|
|
|
|
|
|
|
file, err := os.Create(abs)
|
2015-07-28 09:35:02 +00:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Could not create working directory file: %v", err)
|
|
|
|
}
|
|
|
|
defer file.Close()
|
2017-10-24 17:42:00 +00:00
|
|
|
if _, err := f.Smudge(file, ptr, filename, download, manifest, cb); err != nil {
|
2016-08-18 20:20:33 +00:00
|
|
|
if errors.IsDownloadDeclinedError(err) {
|
2015-08-06 16:12:11 +00:00
|
|
|
// write placeholder data instead
|
2019-07-27 03:42:23 +00:00
|
|
|
file.Seek(0, io.SeekStart)
|
2015-08-06 16:12:11 +00:00
|
|
|
ptr.Encode(file)
|
|
|
|
return err
|
|
|
|
} else {
|
|
|
|
return fmt.Errorf("Could not write working directory file: %v", err)
|
|
|
|
}
|
2015-07-28 09:35:02 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2015-08-27 15:42:23 +00:00
|
|
|
|
2017-11-22 02:00:50 +00:00
|
|
|
func (f *GitFilter) Smudge(writer io.Writer, ptr *Pointer, workingfile string, download bool, manifest *tq.Manifest, cb tools.CopyCallback) (int64, error) {
|
2017-10-25 17:31:15 +00:00
|
|
|
mediafile, err := f.ObjectPath(ptr.Oid)
|
2014-06-05 19:05:02 +00:00
|
|
|
if err != nil {
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, err
|
2014-06-05 19:05:02 +00:00
|
|
|
}
|
2013-10-04 15:22:32 +00:00
|
|
|
|
2017-10-24 21:35:14 +00:00
|
|
|
LinkOrCopyFromReference(f.cfg, ptr.Oid, ptr.Size)
|
2016-01-31 21:06:40 +00:00
|
|
|
|
2016-03-01 19:51:34 +00:00
|
|
|
stat, statErr := os.Stat(mediafile)
|
2015-04-28 15:49:46 +00:00
|
|
|
if statErr == nil && stat != nil {
|
|
|
|
fileSize := stat.Size()
|
2019-10-16 14:57:10 +00:00
|
|
|
if fileSize != ptr.Size {
|
2015-04-28 15:49:46 +00:00
|
|
|
tracerx.Printf("Removing %s, size %d is invalid", mediafile, fileSize)
|
|
|
|
os.RemoveAll(mediafile)
|
|
|
|
stat = nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-31 16:37:28 +00:00
|
|
|
var n int64
|
|
|
|
|
2015-04-28 15:49:46 +00:00
|
|
|
if statErr != nil || stat == nil {
|
2015-08-06 16:12:11 +00:00
|
|
|
if download {
|
2017-10-24 17:42:00 +00:00
|
|
|
n, err = f.downloadFile(writer, ptr, workingfile, mediafile, manifest, cb)
|
2015-08-06 16:12:11 +00:00
|
|
|
} else {
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, errors.NewDownloadDeclinedError(statErr, "smudge")
|
2015-08-06 16:12:11 +00:00
|
|
|
}
|
2013-11-06 16:41:57 +00:00
|
|
|
} else {
|
2017-10-24 17:42:00 +00:00
|
|
|
n, err = f.readLocalFile(writer, ptr, mediafile, workingfile, cb)
|
2013-11-05 15:26:58 +00:00
|
|
|
}
|
|
|
|
|
2015-08-21 18:31:06 +00:00
|
|
|
if err != nil {
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, errors.NewSmudgeError(err, ptr.Oid, mediafile)
|
2014-07-28 19:35:25 +00:00
|
|
|
}
|
2015-05-12 08:45:06 +00:00
|
|
|
|
2017-07-31 16:37:28 +00:00
|
|
|
return n, nil
|
2014-07-28 19:35:25 +00:00
|
|
|
}
|
|
|
|
|
2017-11-22 02:00:50 +00:00
|
|
|
func (f *GitFilter) downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, manifest *tq.Manifest, cb tools.CopyCallback) (int64, error) {
|
2017-07-03 20:40:43 +00:00
|
|
|
fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, humanize.FormatBytes(uint64(ptr.Size)))
|
2014-08-08 17:31:33 +00:00
|
|
|
|
2017-11-22 02:00:50 +00:00
|
|
|
// NOTE: if given, "cb" is a tools.CopyCallback which writes updates
|
2017-08-01 18:58:35 +00:00
|
|
|
// to the logpath specified by GIT_LFS_PROGRESS.
|
|
|
|
//
|
|
|
|
// Either way, forward it into the *tq.TransferQueue so that updates are
|
|
|
|
// sent over correctly.
|
2017-10-30 19:56:02 +00:00
|
|
|
|
2018-01-05 22:01:29 +00:00
|
|
|
q := tq.NewTransferQueue(tq.Download, manifest, f.cfg.Remote(),
|
|
|
|
tq.WithProgressCallback(cb),
|
|
|
|
tq.RemoteRef(f.RemoteRef()),
|
|
|
|
)
|
Don't fail if we lack objects the server has
A Git LFS client may not have the entire history of the objects for the
repository. However, in some situations, we traverse the entire history
of a branch when pushing it, meaning that we need to process every
LFS object in the history of that branch. If the objects for the entire
history are not present, we currently fail to push.
Instead, let's mark objects we don't have on disk as missing and only
fail when we would need to upload those objects. We'll know the server
has the objects if the batch response provides no actions to take for
them when we request an upload. Pass the missing flag down through the
code, and always set it to false for non-uploads.
If for some reason we fail to properly flag a missing object, we will
still fail later on when we cannot open the file, just in a messier and
more poorly controlled way. The technique used here will attempt to
abort the batch as soon as we notice a problem, which means that in the
common case (less than 100 objects) we won't have transferred any
objects, so the user can notice the failure as soon as possible.
Update the tests to look for a string which will occur in the error
message, since we no longer produce the system error message for ENOENT.
2019-04-30 19:18:18 +00:00
|
|
|
q.Add(filepath.Base(workingfile), mediafile, ptr.Oid, ptr.Size, false)
|
2016-12-15 17:13:36 +00:00
|
|
|
q.Wait()
|
|
|
|
|
|
|
|
if errs := q.Errors(); len(errs) > 0 {
|
|
|
|
var multiErr error
|
|
|
|
for _, e := range errs {
|
|
|
|
if multiErr != nil {
|
|
|
|
multiErr = fmt.Errorf("%v\n%v", multiErr, e)
|
|
|
|
} else {
|
|
|
|
multiErr = e
|
|
|
|
}
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, errors.Wrapf(multiErr, "Error downloading %s (%s)", workingfile, ptr.Oid)
|
2016-05-31 09:14:58 +00:00
|
|
|
}
|
2015-10-20 21:36:17 +00:00
|
|
|
}
|
|
|
|
|
2017-10-24 17:42:00 +00:00
|
|
|
return f.readLocalFile(writer, ptr, mediafile, workingfile, nil)
|
2014-07-28 19:35:25 +00:00
|
|
|
}
|
|
|
|
|
2017-11-22 02:00:50 +00:00
|
|
|
func (f *GitFilter) readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile string, cb tools.CopyCallback) (int64, error) {
|
2014-07-28 19:35:25 +00:00
|
|
|
reader, err := os.Open(mediafile)
|
|
|
|
if err != nil {
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, errors.Wrapf(err, "Error opening media file.")
|
2014-07-28 19:35:25 +00:00
|
|
|
}
|
|
|
|
defer reader.Close()
|
|
|
|
|
2014-08-07 16:25:26 +00:00
|
|
|
if ptr.Size == 0 {
|
|
|
|
if stat, _ := os.Stat(mediafile); stat != nil {
|
|
|
|
ptr.Size = stat.Size()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-24 04:53:36 +00:00
|
|
|
if len(ptr.Extensions) > 0 {
|
2017-10-24 17:42:00 +00:00
|
|
|
registeredExts := f.cfg.Extensions()
|
2016-05-13 16:38:06 +00:00
|
|
|
extensions := make(map[string]config.Extension)
|
2015-07-24 04:53:36 +00:00
|
|
|
for _, ptrExt := range ptr.Extensions {
|
|
|
|
ext, ok := registeredExts[ptrExt.Name]
|
|
|
|
if !ok {
|
|
|
|
err := fmt.Errorf("Extension '%s' is not configured.", ptrExt.Name)
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, errors.Wrap(err, "smudge")
|
2015-07-24 04:53:36 +00:00
|
|
|
}
|
|
|
|
ext.Priority = ptrExt.Priority
|
|
|
|
extensions[ext.Name] = ext
|
|
|
|
}
|
2016-05-13 16:38:06 +00:00
|
|
|
exts, err := config.SortExtensions(extensions)
|
2015-07-24 04:53:36 +00:00
|
|
|
if err != nil {
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, errors.Wrap(err, "smudge")
|
2015-07-24 04:53:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// pipe extensions in reverse order
|
2016-05-13 16:38:06 +00:00
|
|
|
var extsR []config.Extension
|
2015-10-20 21:36:17 +00:00
|
|
|
for i := range exts {
|
2015-07-24 04:53:36 +00:00
|
|
|
ext := exts[len(exts)-1-i]
|
|
|
|
extsR = append(extsR, ext)
|
|
|
|
}
|
|
|
|
|
|
|
|
request := &pipeRequest{"smudge", reader, workingfile, extsR}
|
|
|
|
|
2017-10-25 00:59:36 +00:00
|
|
|
response, err := pipeExtensions(f.cfg, request)
|
2015-07-24 04:53:36 +00:00
|
|
|
if err != nil {
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, errors.Wrap(err, "smudge")
|
2015-07-24 04:53:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
actualExts := make(map[string]*pipeExtResult)
|
|
|
|
for _, result := range response.results {
|
|
|
|
actualExts[result.name] = result
|
|
|
|
}
|
|
|
|
|
|
|
|
// verify name, order, and oids
|
|
|
|
oid := response.results[0].oidIn
|
|
|
|
if ptr.Oid != oid {
|
|
|
|
err = fmt.Errorf("Actual oid %s during smudge does not match expected %s", oid, ptr.Oid)
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, errors.Wrap(err, "smudge")
|
2015-07-24 04:53:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, expected := range ptr.Extensions {
|
|
|
|
actual := actualExts[expected.Name]
|
|
|
|
if actual.name != expected.Name {
|
|
|
|
err = fmt.Errorf("Actual extension name '%s' does not match expected '%s'", actual.name, expected.Name)
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, errors.Wrap(err, "smudge")
|
2015-07-24 04:53:36 +00:00
|
|
|
}
|
|
|
|
if actual.oidOut != expected.Oid {
|
|
|
|
err = fmt.Errorf("Actual oid %s for extension '%s' does not match expected %s", actual.oidOut, expected.Name, expected.Oid)
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, errors.Wrap(err, "smudge")
|
2015-07-24 04:53:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// setup reader
|
|
|
|
reader, err = os.Open(response.file.Name())
|
|
|
|
if err != nil {
|
2017-07-31 16:37:28 +00:00
|
|
|
return 0, errors.Wrapf(err, "Error opening smudged file: %s", err)
|
2015-07-24 04:53:36 +00:00
|
|
|
}
|
|
|
|
defer reader.Close()
|
|
|
|
}
|
|
|
|
|
2017-07-31 16:37:28 +00:00
|
|
|
n, err := tools.CopyWithCallback(writer, reader, ptr.Size, cb)
|
2015-07-24 04:53:36 +00:00
|
|
|
if err != nil {
|
2017-07-31 16:37:28 +00:00
|
|
|
return n, errors.Wrapf(err, "Error reading from media file: %s", err)
|
2015-07-24 04:53:36 +00:00
|
|
|
}
|
|
|
|
|
2017-07-31 16:37:28 +00:00
|
|
|
return n, nil
|
2013-10-04 15:22:32 +00:00
|
|
|
}
|