Merge pull request #2459 from git-lfs/windows-size-warning
commands: teach smudge, process-filter to warn about Windows size bug
This commit is contained in:
commit
54c01dbd16
@ -47,8 +47,10 @@ func filterCommand(cmd *cobra.Command, args []string) {
|
|||||||
filter := filepathfilter.New(cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
|
filter := filepathfilter.New(cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
|
||||||
|
|
||||||
var malformed []string
|
var malformed []string
|
||||||
|
var malformedOnWindows []string
|
||||||
|
|
||||||
for s.Scan() {
|
for s.Scan() {
|
||||||
|
var n int64
|
||||||
var err error
|
var err error
|
||||||
var w *git.PktlineWriter
|
var w *git.PktlineWriter
|
||||||
|
|
||||||
@ -62,7 +64,7 @@ func filterCommand(cmd *cobra.Command, args []string) {
|
|||||||
err = clean(w, req.Payload, req.Header["pathname"], -1)
|
err = clean(w, req.Payload, req.Header["pathname"], -1)
|
||||||
case "smudge":
|
case "smudge":
|
||||||
w = git.NewPktlineWriter(os.Stdout, smudgeFilterBufferCapacity)
|
w = git.NewPktlineWriter(os.Stdout, smudgeFilterBufferCapacity)
|
||||||
err = smudge(w, req.Payload, req.Header["pathname"], skip, filter)
|
n, err = smudge(w, req.Payload, req.Header["pathname"], skip, filter)
|
||||||
default:
|
default:
|
||||||
ExitWithError(fmt.Errorf("Unknown command %q", req.Header["command"]))
|
ExitWithError(fmt.Errorf("Unknown command %q", req.Header["command"]))
|
||||||
}
|
}
|
||||||
@ -70,6 +72,8 @@ func filterCommand(cmd *cobra.Command, args []string) {
|
|||||||
if errors.IsNotAPointerError(err) {
|
if errors.IsNotAPointerError(err) {
|
||||||
malformed = append(malformed, req.Header["pathname"])
|
malformed = append(malformed, req.Header["pathname"])
|
||||||
err = nil
|
err = nil
|
||||||
|
} else if possiblyMalformedSmudge(n) {
|
||||||
|
malformedOnWindows = append(malformedOnWindows, req.Header["pathname"])
|
||||||
}
|
}
|
||||||
|
|
||||||
var status string
|
var status string
|
||||||
@ -89,6 +93,16 @@ func filterCommand(cmd *cobra.Command, args []string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(malformedOnWindows) > 0 {
|
||||||
|
fmt.Fprintf(os.Stderr, "Encountered %d file(s) that may not have been copied correctly on Windows:\n")
|
||||||
|
|
||||||
|
for _, m := range malformedOnWindows {
|
||||||
|
fmt.Fprintf(os.Stderr, "\t%s\n", m)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(os.Stderr, "\nSee: `git lfs help smudge` for more details.\n")
|
||||||
|
}
|
||||||
|
|
||||||
if err := s.Err(); err != nil && err != io.EOF {
|
if err := s.Err(); err != nil && err != io.EOF {
|
||||||
ExitWithError(err)
|
ExitWithError(err)
|
||||||
}
|
}
|
||||||
|
@ -4,12 +4,14 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"runtime"
|
||||||
|
|
||||||
"github.com/git-lfs/git-lfs/errors"
|
"github.com/git-lfs/git-lfs/errors"
|
||||||
"github.com/git-lfs/git-lfs/filepathfilter"
|
"github.com/git-lfs/git-lfs/filepathfilter"
|
||||||
"github.com/git-lfs/git-lfs/lfs"
|
"github.com/git-lfs/git-lfs/lfs"
|
||||||
"github.com/git-lfs/git-lfs/localstorage"
|
"github.com/git-lfs/git-lfs/localstorage"
|
||||||
"github.com/git-lfs/git-lfs/tools"
|
"github.com/git-lfs/git-lfs/tools"
|
||||||
|
"github.com/git-lfs/git-lfs/tools/humanize"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -34,26 +36,26 @@ var (
|
|||||||
// Any errors encountered along the way will be returned immediately if they
|
// Any errors encountered along the way will be returned immediately if they
|
||||||
// were non-fatal, otherwise execution will halt and the process will be
|
// were non-fatal, otherwise execution will halt and the process will be
|
||||||
// terminated by using the `commands.Panic()` func.
|
// terminated by using the `commands.Panic()` func.
|
||||||
func smudge(to io.Writer, from io.Reader, filename string, skip bool, filter *filepathfilter.Filter) error {
|
func smudge(to io.Writer, from io.Reader, filename string, skip bool, filter *filepathfilter.Filter) (int64, error) {
|
||||||
ptr, pbuf, perr := lfs.DecodeFrom(from)
|
ptr, pbuf, perr := lfs.DecodeFrom(from)
|
||||||
if perr != nil {
|
if perr != nil {
|
||||||
n, err := tools.Spool(to, pbuf, localstorage.Objects().TempDir)
|
n, err := tools.Spool(to, pbuf, localstorage.Objects().TempDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, perr.Error())
|
return 0, errors.Wrap(err, perr.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
if n != 0 {
|
if n != 0 {
|
||||||
return errors.NewNotAPointerError(errors.Errorf(
|
return 0, errors.NewNotAPointerError(errors.Errorf(
|
||||||
"Unable to parse pointer at: %q", filename,
|
"Unable to parse pointer at: %q", filename,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
return nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
lfs.LinkOrCopyFromReference(ptr.Oid, ptr.Size)
|
lfs.LinkOrCopyFromReference(ptr.Oid, ptr.Size)
|
||||||
cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1)
|
cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
download := !skip
|
download := !skip
|
||||||
@ -61,7 +63,7 @@ func smudge(to io.Writer, from io.Reader, filename string, skip bool, filter *fi
|
|||||||
download = filter.Allows(filename)
|
download = filter.Allows(filename)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = ptr.Smudge(to, filename, download, getTransferManifest(), cb)
|
n, err := ptr.Smudge(to, filename, download, getTransferManifest(), cb)
|
||||||
if file != nil {
|
if file != nil {
|
||||||
file.Close()
|
file.Close()
|
||||||
}
|
}
|
||||||
@ -82,7 +84,7 @@ func smudge(to io.Writer, from io.Reader, filename string, skip bool, filter *fi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func smudgeCommand(cmd *cobra.Command, args []string) {
|
func smudgeCommand(cmd *cobra.Command, args []string) {
|
||||||
@ -94,12 +96,14 @@ func smudgeCommand(cmd *cobra.Command, args []string) {
|
|||||||
}
|
}
|
||||||
filter := filepathfilter.New(cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
|
filter := filepathfilter.New(cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
|
||||||
|
|
||||||
if err := smudge(os.Stdout, os.Stdin, smudgeFilename(args), smudgeSkip, filter); err != nil {
|
if n, err := smudge(os.Stdout, os.Stdin, smudgeFilename(args), smudgeSkip, filter); err != nil {
|
||||||
if errors.IsNotAPointerError(err) {
|
if errors.IsNotAPointerError(err) {
|
||||||
fmt.Fprintln(os.Stderr, err.Error())
|
fmt.Fprintln(os.Stderr, err.Error())
|
||||||
} else {
|
} else {
|
||||||
Error(err.Error())
|
Error(err.Error())
|
||||||
}
|
}
|
||||||
|
} else if possiblyMalformedSmudge(n) {
|
||||||
|
fmt.Fprintln(os.Stderr, "Possibly malformed smudge on Windows: see `git lfs help smudge` for more info.")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -110,6 +114,10 @@ func smudgeFilename(args []string) string {
|
|||||||
return "<unknown file>"
|
return "<unknown file>"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func possiblyMalformedSmudge(n int64) bool {
|
||||||
|
return n > 4*humanize.Gigabyte && runtime.GOOS == "windows"
|
||||||
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
RegisterCommand("smudge", smudgeCommand, func(cmd *cobra.Command) {
|
RegisterCommand("smudge", smudgeCommand, func(cmd *cobra.Command) {
|
||||||
cmd.Flags().BoolVarP(&smudgeSkip, "skip", "s", false, "")
|
cmd.Flags().BoolVarP(&smudgeSkip, "skip", "s", false, "")
|
||||||
|
@ -24,6 +24,13 @@ standard output.
|
|||||||
* `--skip`:
|
* `--skip`:
|
||||||
Skip automatic downloading of objects on clone or pull.
|
Skip automatic downloading of objects on clone or pull.
|
||||||
|
|
||||||
|
## KNOWN BUGS
|
||||||
|
|
||||||
|
On Windows, Git does not handle files in the working tree larger than 4
|
||||||
|
gigabytes.
|
||||||
|
|
||||||
|
For more information, see: https://github.com/git-lfs/git-lfs/issues/2434.
|
||||||
|
|
||||||
## SEE ALSO
|
## SEE ALSO
|
||||||
|
|
||||||
git-lfs-install(1), gitattributes(5).
|
git-lfs-install(1), gitattributes(5).
|
||||||
|
@ -60,7 +60,7 @@ func NewPointerExtension(name string, priority int, oid string) *PointerExtensio
|
|||||||
return &PointerExtension{name, priority, oid, oidType}
|
return &PointerExtension{name, priority, oid, oidType}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *Pointer) Smudge(writer io.Writer, workingfile string, download bool, manifest *tq.Manifest, cb progress.CopyCallback) error {
|
func (p *Pointer) Smudge(writer io.Writer, workingfile string, download bool, manifest *tq.Manifest, cb progress.CopyCallback) (int64, error) {
|
||||||
return PointerSmudge(writer, p, workingfile, download, manifest, cb)
|
return PointerSmudge(writer, p, workingfile, download, manifest, cb)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -23,7 +23,7 @@ func PointerSmudgeToFile(filename string, ptr *Pointer, download bool, manifest
|
|||||||
return fmt.Errorf("Could not create working directory file: %v", err)
|
return fmt.Errorf("Could not create working directory file: %v", err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
if err := PointerSmudge(file, ptr, filename, download, manifest, cb); err != nil {
|
if _, err := PointerSmudge(file, ptr, filename, download, manifest, cb); err != nil {
|
||||||
if errors.IsDownloadDeclinedError(err) {
|
if errors.IsDownloadDeclinedError(err) {
|
||||||
// write placeholder data instead
|
// write placeholder data instead
|
||||||
file.Seek(0, os.SEEK_SET)
|
file.Seek(0, os.SEEK_SET)
|
||||||
@ -36,10 +36,10 @@ func PointerSmudgeToFile(filename string, ptr *Pointer, download bool, manifest
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download bool, manifest *tq.Manifest, cb progress.CopyCallback) error {
|
func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download bool, manifest *tq.Manifest, cb progress.CopyCallback) (int64, error) {
|
||||||
mediafile, err := LocalMediaPath(ptr.Oid)
|
mediafile, err := LocalMediaPath(ptr.Oid)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
LinkOrCopyFromReference(ptr.Oid, ptr.Size)
|
LinkOrCopyFromReference(ptr.Oid, ptr.Size)
|
||||||
@ -54,24 +54,26 @@ func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var n int64
|
||||||
|
|
||||||
if statErr != nil || stat == nil {
|
if statErr != nil || stat == nil {
|
||||||
if download {
|
if download {
|
||||||
err = downloadFile(writer, ptr, workingfile, mediafile, manifest, cb)
|
n, err = downloadFile(writer, ptr, workingfile, mediafile, manifest, cb)
|
||||||
} else {
|
} else {
|
||||||
return errors.NewDownloadDeclinedError(statErr, "smudge")
|
return 0, errors.NewDownloadDeclinedError(statErr, "smudge")
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
err = readLocalFile(writer, ptr, mediafile, workingfile, cb)
|
n, err = readLocalFile(writer, ptr, mediafile, workingfile, cb)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.NewSmudgeError(err, ptr.Oid, mediafile)
|
return 0, errors.NewSmudgeError(err, ptr.Oid, mediafile)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, manifest *tq.Manifest, cb progress.CopyCallback) error {
|
func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, manifest *tq.Manifest, cb progress.CopyCallback) (int64, error) {
|
||||||
fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, humanize.FormatBytes(uint64(ptr.Size)))
|
fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, humanize.FormatBytes(uint64(ptr.Size)))
|
||||||
|
|
||||||
q := tq.NewTransferQueue(tq.Download, manifest, "")
|
q := tq.NewTransferQueue(tq.Download, manifest, "")
|
||||||
@ -86,17 +88,17 @@ func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string,
|
|||||||
} else {
|
} else {
|
||||||
multiErr = e
|
multiErr = e
|
||||||
}
|
}
|
||||||
return errors.Wrapf(multiErr, "Error downloading %s (%s)", workingfile, ptr.Oid)
|
return 0, errors.Wrapf(multiErr, "Error downloading %s (%s)", workingfile, ptr.Oid)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return readLocalFile(writer, ptr, mediafile, workingfile, nil)
|
return readLocalFile(writer, ptr, mediafile, workingfile, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile string, cb progress.CopyCallback) error {
|
func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile string, cb progress.CopyCallback) (int64, error) {
|
||||||
reader, err := os.Open(mediafile)
|
reader, err := os.Open(mediafile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrapf(err, "Error opening media file.")
|
return 0, errors.Wrapf(err, "Error opening media file.")
|
||||||
}
|
}
|
||||||
defer reader.Close()
|
defer reader.Close()
|
||||||
|
|
||||||
@ -113,14 +115,14 @@ func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile
|
|||||||
ext, ok := registeredExts[ptrExt.Name]
|
ext, ok := registeredExts[ptrExt.Name]
|
||||||
if !ok {
|
if !ok {
|
||||||
err := fmt.Errorf("Extension '%s' is not configured.", ptrExt.Name)
|
err := fmt.Errorf("Extension '%s' is not configured.", ptrExt.Name)
|
||||||
return errors.Wrap(err, "smudge")
|
return 0, errors.Wrap(err, "smudge")
|
||||||
}
|
}
|
||||||
ext.Priority = ptrExt.Priority
|
ext.Priority = ptrExt.Priority
|
||||||
extensions[ext.Name] = ext
|
extensions[ext.Name] = ext
|
||||||
}
|
}
|
||||||
exts, err := config.SortExtensions(extensions)
|
exts, err := config.SortExtensions(extensions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "smudge")
|
return 0, errors.Wrap(err, "smudge")
|
||||||
}
|
}
|
||||||
|
|
||||||
// pipe extensions in reverse order
|
// pipe extensions in reverse order
|
||||||
@ -134,7 +136,7 @@ func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile
|
|||||||
|
|
||||||
response, err := pipeExtensions(request)
|
response, err := pipeExtensions(request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "smudge")
|
return 0, errors.Wrap(err, "smudge")
|
||||||
}
|
}
|
||||||
|
|
||||||
actualExts := make(map[string]*pipeExtResult)
|
actualExts := make(map[string]*pipeExtResult)
|
||||||
@ -146,33 +148,33 @@ func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile
|
|||||||
oid := response.results[0].oidIn
|
oid := response.results[0].oidIn
|
||||||
if ptr.Oid != oid {
|
if ptr.Oid != oid {
|
||||||
err = fmt.Errorf("Actual oid %s during smudge does not match expected %s", oid, ptr.Oid)
|
err = fmt.Errorf("Actual oid %s during smudge does not match expected %s", oid, ptr.Oid)
|
||||||
return errors.Wrap(err, "smudge")
|
return 0, errors.Wrap(err, "smudge")
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, expected := range ptr.Extensions {
|
for _, expected := range ptr.Extensions {
|
||||||
actual := actualExts[expected.Name]
|
actual := actualExts[expected.Name]
|
||||||
if actual.name != expected.Name {
|
if actual.name != expected.Name {
|
||||||
err = fmt.Errorf("Actual extension name '%s' does not match expected '%s'", actual.name, expected.Name)
|
err = fmt.Errorf("Actual extension name '%s' does not match expected '%s'", actual.name, expected.Name)
|
||||||
return errors.Wrap(err, "smudge")
|
return 0, errors.Wrap(err, "smudge")
|
||||||
}
|
}
|
||||||
if actual.oidOut != expected.Oid {
|
if actual.oidOut != expected.Oid {
|
||||||
err = fmt.Errorf("Actual oid %s for extension '%s' does not match expected %s", actual.oidOut, expected.Name, expected.Oid)
|
err = fmt.Errorf("Actual oid %s for extension '%s' does not match expected %s", actual.oidOut, expected.Name, expected.Oid)
|
||||||
return errors.Wrap(err, "smudge")
|
return 0, errors.Wrap(err, "smudge")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// setup reader
|
// setup reader
|
||||||
reader, err = os.Open(response.file.Name())
|
reader, err = os.Open(response.file.Name())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrapf(err, "Error opening smudged file: %s", err)
|
return 0, errors.Wrapf(err, "Error opening smudged file: %s", err)
|
||||||
}
|
}
|
||||||
defer reader.Close()
|
defer reader.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = tools.CopyWithCallback(writer, reader, ptr.Size, cb)
|
n, err := tools.CopyWithCallback(writer, reader, ptr.Size, cb)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrapf(err, "Error reading from media file: %s", err)
|
return n, errors.Wrapf(err, "Error reading from media file: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user