lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2459 from git-lfs/windows-size-warning

Browse Source 
commands: teach smudge, process-filter to warn about Windows size bug
This commit is contained in:
Taylor Blau 2017-07-31 13:24:14 -06:00 committed by GitHub
commit 54c01dbd16
5 changed files with 63 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
commands/command_filter_process.go
View File

@ -47,8 +47,10 @@ func filterCommand(cmd *cobra.Command, args []string) {
filter := filepathfilter.New(cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
var malformed []string
var malformedOnWindows []string
for s.Scan() {
var n int64
var err error
var w *git.PktlineWriter
@ -62,7 +64,7 @@ func filterCommand(cmd *cobra.Command, args []string) {
err = clean(w, req.Payload, req.Header["pathname"], -1)
case "smudge":
w = git.NewPktlineWriter(os.Stdout, smudgeFilterBufferCapacity)
err = smudge(w, req.Payload, req.Header["pathname"], skip, filter)
n, err = smudge(w, req.Payload, req.Header["pathname"], skip, filter)
default:
ExitWithError(fmt.Errorf("Unknown command %q", req.Header["command"]))
}
@ -70,6 +72,8 @@ func filterCommand(cmd *cobra.Command, args []string) {
if errors.IsNotAPointerError(err) {
malformed = append(malformed, req.Header["pathname"])
err = nil
} else if possiblyMalformedSmudge(n) {
malformedOnWindows = append(malformedOnWindows, req.Header["pathname"])
}
var status string
@ -89,6 +93,16 @@ func filterCommand(cmd *cobra.Command, args []string) {
}
}
if len(malformedOnWindows) > 0 {
fmt.Fprintf(os.Stderr, "Encountered %d file(s) that may not have been copied correctly on Windows:\n")
for _, m := range malformedOnWindows {
fmt.Fprintf(os.Stderr, "\t%s\n", m)
}
fmt.Fprintf(os.Stderr, "\nSee: `git lfs help smudge` for more details.\n")
}
if err := s.Err(); err != nil && err != io.EOF {
ExitWithError(err)
}

24
commands/command_smudge.go
View File

@ -4,12 +4,14 @@ import (
"fmt"
"io"
"os"
"runtime"
"github.com/git-lfs/git-lfs/errors"
"github.com/git-lfs/git-lfs/filepathfilter"
"github.com/git-lfs/git-lfs/lfs"
"github.com/git-lfs/git-lfs/localstorage"
"github.com/git-lfs/git-lfs/tools"
"github.com/git-lfs/git-lfs/tools/humanize"
"github.com/spf13/cobra"
)
@ -34,26 +36,26 @@ var (
// Any errors encountered along the way will be returned immediately if they
// were non-fatal, otherwise execution will halt and the process will be
// terminated by using the `commands.Panic()` func.
func smudge(to io.Writer, from io.Reader, filename string, skip bool, filter *filepathfilter.Filter) error {
func smudge(to io.Writer, from io.Reader, filename string, skip bool, filter *filepathfilter.Filter) (int64, error) {
ptr, pbuf, perr := lfs.DecodeFrom(from)
if perr != nil {
n, err := tools.Spool(to, pbuf, localstorage.Objects().TempDir)
if err != nil {
return errors.Wrap(err, perr.Error())
return 0, errors.Wrap(err, perr.Error())
}
if n != 0 {
return errors.NewNotAPointerError(errors.Errorf(
return 0, errors.NewNotAPointerError(errors.Errorf(
"Unable to parse pointer at: %q", filename,
))
}
return nil
return 0, nil
}
lfs.LinkOrCopyFromReference(ptr.Oid, ptr.Size)
cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1)
if err != nil {
return err
return 0, err
}
download := !skip
@ -61,7 +63,7 @@ func smudge(to io.Writer, from io.Reader, filename string, skip bool, filter *fi
download = filter.Allows(filename)
}
err = ptr.Smudge(to, filename, download, getTransferManifest(), cb)
n, err := ptr.Smudge(to, filename, download, getTransferManifest(), cb)
if file != nil {
file.Close()
}
@ -82,7 +84,7 @@ func smudge(to io.Writer, from io.Reader, filename string, skip bool, filter *fi
}
}
return nil
return n, nil
}
func smudgeCommand(cmd *cobra.Command, args []string) {
@ -94,12 +96,14 @@ func smudgeCommand(cmd *cobra.Command, args []string) {
}
filter := filepathfilter.New(cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
if err := smudge(os.Stdout, os.Stdin, smudgeFilename(args), smudgeSkip, filter); err != nil {
if n, err := smudge(os.Stdout, os.Stdin, smudgeFilename(args), smudgeSkip, filter); err != nil {
if errors.IsNotAPointerError(err) {
fmt.Fprintln(os.Stderr, err.Error())
} else {
Error(err.Error())
}
} else if possiblyMalformedSmudge(n) {
fmt.Fprintln(os.Stderr, "Possibly malformed smudge on Windows: see `git lfs help smudge` for more info.")
}
}
@ -110,6 +114,10 @@ func smudgeFilename(args []string) string {
return "<unknown file>"
}
func possiblyMalformedSmudge(n int64) bool {
return n > 4*humanize.Gigabyte && runtime.GOOS == "windows"
}
func init() {
RegisterCommand("smudge", smudgeCommand, func(cmd *cobra.Command) {
cmd.Flags().BoolVarP(&smudgeSkip, "skip", "s", false, "")

7
docs/man/git-lfs-smudge.1.ronn
View File

@ -24,6 +24,13 @@ standard output.
* `--skip`:
Skip automatic downloading of objects on clone or pull.
## KNOWN BUGS
On Windows, Git does not handle files in the working tree larger than 4
gigabytes.
For more information, see: https://github.com/git-lfs/git-lfs/issues/2434.
## SEE ALSO
git-lfs-install(1), gitattributes(5).

2
lfs/pointer.go
View File

@ -60,7 +60,7 @@ func NewPointerExtension(name string, priority int, oid string) *PointerExtensio
return &PointerExtension{name, priority, oid, oidType}
}
func (p *Pointer) Smudge(writer io.Writer, workingfile string, download bool, manifest *tq.Manifest, cb progress.CopyCallback) error {
func (p *Pointer) Smudge(writer io.Writer, workingfile string, download bool, manifest *tq.Manifest, cb progress.CopyCallback) (int64, error) {
return PointerSmudge(writer, p, workingfile, download, manifest, cb)
}

46
lfs/pointer_smudge.go
View File

@ -23,7 +23,7 @@ func PointerSmudgeToFile(filename string, ptr *Pointer, download bool, manifest
return fmt.Errorf("Could not create working directory file: %v", err)
}
defer file.Close()
if err := PointerSmudge(file, ptr, filename, download, manifest, cb); err != nil {
if _, err := PointerSmudge(file, ptr, filename, download, manifest, cb); err != nil {
if errors.IsDownloadDeclinedError(err) {
// write placeholder data instead
file.Seek(0, os.SEEK_SET)
@ -36,10 +36,10 @@ func PointerSmudgeToFile(filename string, ptr *Pointer, download bool, manifest
return nil
}
func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download bool, manifest *tq.Manifest, cb progress.CopyCallback) error {
func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download bool, manifest *tq.Manifest, cb progress.CopyCallback) (int64, error) {
mediafile, err := LocalMediaPath(ptr.Oid)
if err != nil {
return err
return 0, err
}
LinkOrCopyFromReference(ptr.Oid, ptr.Size)
@ -54,24 +54,26 @@ func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download
}
}
var n int64
if statErr != nil || stat == nil {
if download {
err = downloadFile(writer, ptr, workingfile, mediafile, manifest, cb)
n, err = downloadFile(writer, ptr, workingfile, mediafile, manifest, cb)
} else {
return errors.NewDownloadDeclinedError(statErr, "smudge")
return 0, errors.NewDownloadDeclinedError(statErr, "smudge")
}
} else {
err = readLocalFile(writer, ptr, mediafile, workingfile, cb)
n, err = readLocalFile(writer, ptr, mediafile, workingfile, cb)
}
if err != nil {
return errors.NewSmudgeError(err, ptr.Oid, mediafile)
return 0, errors.NewSmudgeError(err, ptr.Oid, mediafile)
}
return nil
return n, nil
}
func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, manifest *tq.Manifest, cb progress.CopyCallback) error {
func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, manifest *tq.Manifest, cb progress.CopyCallback) (int64, error) {
fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, humanize.FormatBytes(uint64(ptr.Size)))
q := tq.NewTransferQueue(tq.Download, manifest, "")
@ -86,17 +88,17 @@ func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string,
} else {
multiErr = e
}
return errors.Wrapf(multiErr, "Error downloading %s (%s)", workingfile, ptr.Oid)
return 0, errors.Wrapf(multiErr, "Error downloading %s (%s)", workingfile, ptr.Oid)
}
}
return readLocalFile(writer, ptr, mediafile, workingfile, nil)
}
func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile string, cb progress.CopyCallback) error {
func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile string, cb progress.CopyCallback) (int64, error) {
reader, err := os.Open(mediafile)
if err != nil {
return errors.Wrapf(err, "Error opening media file.")
return 0, errors.Wrapf(err, "Error opening media file.")
}
defer reader.Close()
@ -113,14 +115,14 @@ func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile
ext, ok := registeredExts[ptrExt.Name]
if !ok {
err := fmt.Errorf("Extension '%s' is not configured.", ptrExt.Name)
return errors.Wrap(err, "smudge")
return 0, errors.Wrap(err, "smudge")
}
ext.Priority = ptrExt.Priority
extensions[ext.Name] = ext
}
exts, err := config.SortExtensions(extensions)
if err != nil {
return errors.Wrap(err, "smudge")
return 0, errors.Wrap(err, "smudge")
}
// pipe extensions in reverse order
@ -134,7 +136,7 @@ func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile
response, err := pipeExtensions(request)
if err != nil {
return errors.Wrap(err, "smudge")
return 0, errors.Wrap(err, "smudge")
}
actualExts := make(map[string]*pipeExtResult)
@ -146,33 +148,33 @@ func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile
oid := response.results[0].oidIn
if ptr.Oid != oid {
err = fmt.Errorf("Actual oid %s during smudge does not match expected %s", oid, ptr.Oid)
return errors.Wrap(err, "smudge")
return 0, errors.Wrap(err, "smudge")
}
for _, expected := range ptr.Extensions {
actual := actualExts[expected.Name]
if actual.name != expected.Name {
err = fmt.Errorf("Actual extension name '%s' does not match expected '%s'", actual.name, expected.Name)
return errors.Wrap(err, "smudge")
return 0, errors.Wrap(err, "smudge")
}
if actual.oidOut != expected.Oid {
err = fmt.Errorf("Actual oid %s for extension '%s' does not match expected %s", actual.oidOut, expected.Name, expected.Oid)
return errors.Wrap(err, "smudge")
return 0, errors.Wrap(err, "smudge")
}
}
// setup reader
reader, err = os.Open(response.file.Name())
if err != nil {
return errors.Wrapf(err, "Error opening smudged file: %s", err)
return 0, errors.Wrapf(err, "Error opening smudged file: %s", err)
}
defer reader.Close()
}
_, err = tools.CopyWithCallback(writer, reader, ptr.Size, cb)
n, err := tools.CopyWithCallback(writer, reader, ptr.Size, cb)
if err != nil {
return errors.Wrapf(err, "Error reading from media file: %s", err)
return n, errors.Wrapf(err, "Error reading from media file: %s", err)
}
return nil
return n, nil
}