diff --git a/commands/command_pointer.go b/commands/command_pointer.go index 70294dfd..c6595283 100644 --- a/commands/command_pointer.go +++ b/commands/command_pointer.go @@ -8,8 +8,8 @@ import ( "fmt" "io" "os" - "os/exec" + "github.com/git-lfs/git-lfs/git" "github.com/git-lfs/git-lfs/lfs" "github.com/spf13/cobra" ) @@ -53,7 +53,11 @@ func pointerCommand(cmd *cobra.Command, args []string) { lfs.EncodePointer(io.MultiWriter(os.Stdout, buf), ptr) if comparing { - buildOid = gitHashObject(buf.Bytes()) + buildOid, err = git.HashObject(bytes.NewReader(buf.Bytes())) + if err != nil { + Error(err.Error()) + os.Exit(1) + } fmt.Fprintf(os.Stderr, "\nGit blob OID: %s\n\n", buildOid) } } else { @@ -86,7 +90,11 @@ func pointerCommand(cmd *cobra.Command, args []string) { fmt.Fprintf(os.Stderr, buf.String()) if comparing { - compareOid = gitHashObject(buf.Bytes()) + compareOid, err = git.HashObject(bytes.NewReader(buf.Bytes())) + if err != nil { + Error(err.Error()) + os.Exit(1) + } fmt.Fprintf(os.Stderr, "\nGit blob OID: %s\n", compareOid) } } @@ -116,18 +124,6 @@ func pointerReader() (io.ReadCloser, error) { return os.Stdin, nil } -func gitHashObject(by []byte) string { - cmd := exec.Command("git", "hash-object", "--stdin") - cmd.Stdin = bytes.NewReader(by) - out, err := cmd.Output() - if err != nil { - Error("Error building Git blob OID: %s", err) - os.Exit(1) - } - - return string(bytes.TrimSpace(out)) -} - func init() { RegisterCommand("pointer", pointerCommand, func(cmd *cobra.Command) { cmd.Flags().StringVarP(&pointerFile, "file", "f", "", "Path to a local file to generate the pointer from.") diff --git a/commands/pull.go b/commands/pull.go index 0d1e6286..5f45803f 100644 --- a/commands/pull.go +++ b/commands/pull.go @@ -5,11 +5,12 @@ import ( "fmt" "io" "os" - "os/exec" "sync" "github.com/git-lfs/git-lfs/errors" + "github.com/git-lfs/git-lfs/git" "github.com/git-lfs/git-lfs/lfs" + "github.com/git-lfs/git-lfs/subprocess" "github.com/git-lfs/git-lfs/tq" ) @@ -85,7 +86,7 @@ func (c *singleCheckout) Close() { // which can trigger entire working copy to be re-examined, which triggers clean filters // and which has unexpected side effects (e.g. downloading filtered-out files) type gitIndexer struct { - cmd *exec.Cmd + cmd *subprocess.Cmd input io.WriteCloser output bytes.Buffer mu sync.Mutex @@ -97,14 +98,7 @@ func (i *gitIndexer) Add(path string) error { if i.cmd == nil { // Fire up the update-index command - i.cmd = exec.Command("git", "update-index", "-q", "--refresh", "--stdin") - i.cmd.Stdout = &i.output - i.cmd.Stderr = &i.output - stdin, err := i.cmd.StdinPipe() - if err == nil { - err = i.cmd.Start() - } - + stdin, err := git.StartUpdateIndexFromStdin(&i.output) if err != nil { return err } diff --git a/git/git.go b/git/git.go index 772d52be..6b593d9a 100644 --- a/git/git.go +++ b/git/git.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "errors" "fmt" + "io" "io/ioutil" "net/url" "os" @@ -84,19 +85,116 @@ type CommitSummary struct { Subject string } +// Prepend Git config instructions to disable Git LFS filter +func gitConfigNoLFS(args ...string) []string { + // Before git 2.8, setting filters to blank causes lots of warnings, so use cat instead (slightly slower) + // Also pre 2.2 it failed completely. We used to use it anyway in git 2.2-2.7 and + // suppress the messages in stderr, but doing that with standard StderrPipe suppresses + // the git clone output (git thinks it's not a terminal) and makes it look like it's + // not working. You can get around that with https://github.com/kr/pty but that + // causes difficult issues with passing through Stdin for login prompts + // This way is simpler & more practical. + filterOverride := "" + if !Config.IsGitVersionAtLeast("2.8.0") { + filterOverride = "cat" + } + + return append([]string{ + "-c", fmt.Sprintf("filter.lfs.smudge=%v", filterOverride), + "-c", fmt.Sprintf("filter.lfs.clean=%v", filterOverride), + "-c", "filter.lfs.process=", + "-c", "filter.lfs.required=false", + }, args...) +} + +// Invoke Git with disabled LFS filters +func gitNoLFS(args ...string) *subprocess.Cmd { + return subprocess.ExecCommand("git", gitConfigNoLFS(args...)...) +} + +func gitNoLFSSimple(args ...string) (string, error) { + return subprocess.SimpleExec("git", gitConfigNoLFS(args...)...) +} + +func gitNoLFSBuffered(args ...string) (*subprocess.BufferedCmd, error) { + return subprocess.BufferedExec("git", gitConfigNoLFS(args...)...) +} + +// Invoke Git with enabled LFS filters +func git(args ...string) *subprocess.Cmd { + return subprocess.ExecCommand("git", args...) +} + +func gitSimple(args ...string) (string, error) { + return subprocess.SimpleExec("git", args...) +} + +func gitBuffered(args ...string) (*subprocess.BufferedCmd, error) { + return subprocess.BufferedExec("git", args...) +} + +func CatFile() (*subprocess.BufferedCmd, error) { + return gitNoLFSBuffered("cat-file", "--batch-check") +} + +func DiffIndex(ref string, cached bool) (*bufio.Scanner, error) { + args := []string{"diff-index", "-M"} + if cached { + args = append(args, "--cached") + } + args = append(args, ref) + + cmd, err := gitBuffered(args...) + if err != nil { + return nil, err + } + if err = cmd.Stdin.Close(); err != nil { + return nil, err + } + + return bufio.NewScanner(cmd.Stdout), nil +} + +func HashObject(r io.Reader) (string, error) { + cmd := gitNoLFS("hash-object", "--stdin") + cmd.Stdin = r + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("Error building Git blob OID: %s", err) + } + + return string(bytes.TrimSpace(out)), nil +} + +func Log(args ...string) (*subprocess.BufferedCmd, error) { + logArgs := append([]string{"log"}, args...) + return gitNoLFSBuffered(logArgs...) +} + func LsRemote(remote, remoteRef string) (string, error) { if remote == "" { return "", errors.New("remote required") } if remoteRef == "" { - return subprocess.SimpleExec("git", "ls-remote", remote) + return gitNoLFSSimple("ls-remote", remote) } - return subprocess.SimpleExec("git", "ls-remote", remote, remoteRef) + return gitNoLFSSimple("ls-remote", remote, remoteRef) +} + +func LsTree(ref string) (*subprocess.BufferedCmd, error) { + return gitNoLFSBuffered( + "ls-tree", + "-r", // recurse + "-l", // report object size (we'll need this) + "-z", // null line termination + "--full-tree", // start at the root regardless of where we are in it + ref, + ) } func ResolveRef(ref string) (*Ref, error) { - outp, err := subprocess.SimpleExec("git", "rev-parse", ref, "--symbolic-full-name", ref) + outp, err := gitNoLFSSimple("rev-parse", ref, "--symbolic-full-name", ref) if err != nil { return nil, fmt.Errorf("Git can't resolve ref: %q", ref) } @@ -199,7 +297,7 @@ func RemoteBranchForLocalBranch(localBranch string) string { } func RemoteList() ([]string, error) { - cmd := subprocess.ExecCommand("git", "remote") + cmd := gitNoLFS("remote") outp, err := cmd.StdoutPipe() if err != nil { @@ -221,7 +319,7 @@ func RemoteList() ([]string, error) { // Refs returns all of the local and remote branches and tags for the current // repository. Other refs (HEAD, refs/stash, git notes) are ignored. func LocalRefs() ([]*Ref, error) { - cmd := subprocess.ExecCommand("git", "show-ref", "--heads", "--tags") + cmd := gitNoLFS("show-ref", "--heads", "--tags") outp, err := cmd.StdoutPipe() if err != nil { @@ -277,7 +375,7 @@ func UpdateRefIn(wd string, ref *Ref, to []byte, reason string) error { args = append(args, "-m", reason) } - cmd := subprocess.ExecCommand("git", args...) + cmd := gitNoLFS(args...) cmd.Dir = wd return cmd.Run() @@ -360,9 +458,16 @@ func DefaultRemote() (string, error) { return "", errors.New("Unable to pick default remote, too ambiguous") } -func UpdateIndex(file string) error { - _, err := subprocess.SimpleExec("git", "update-index", "-q", "--refresh", file) - return err +func StartUpdateIndexFromStdin(w io.Writer) (io.WriteCloser, error) { + cmd := gitNoLFS("update-index", "-q", "--refresh", "--stdin") + cmd.Stdout = w + cmd.Stderr = w + stdin, err := cmd.StdinPipe() + if err == nil { + err = cmd.Start() + } + + return stdin, err } type gitConfig struct { @@ -374,61 +479,61 @@ var Config = &gitConfig{} // Find returns the git config value for the key func (c *gitConfig) Find(val string) string { - output, _ := subprocess.SimpleExec("git", "config", val) + output, _ := gitSimple("config", val) return output } // FindGlobal returns the git config value global scope for the key func (c *gitConfig) FindGlobal(val string) string { - output, _ := subprocess.SimpleExec("git", "config", "--global", val) + output, _ := gitSimple("config", "--global", val) return output } // FindSystem returns the git config value in system scope for the key func (c *gitConfig) FindSystem(val string) string { - output, _ := subprocess.SimpleExec("git", "config", "--system", val) + output, _ := gitSimple("config", "--system", val) return output } // Find returns the git config value for the key func (c *gitConfig) FindLocal(val string) string { - output, _ := subprocess.SimpleExec("git", "config", "--local", val) + output, _ := gitSimple("config", "--local", val) return output } // SetGlobal sets the git config value for the key in the global config func (c *gitConfig) SetGlobal(key, val string) (string, error) { - return subprocess.SimpleExec("git", "config", "--global", key, val) + return gitSimple("config", "--global", key, val) } // SetSystem sets the git config value for the key in the system config func (c *gitConfig) SetSystem(key, val string) (string, error) { - return subprocess.SimpleExec("git", "config", "--system", key, val) + return gitSimple("config", "--system", key, val) } // UnsetGlobal removes the git config value for the key from the global config func (c *gitConfig) UnsetGlobal(key string) (string, error) { - return subprocess.SimpleExec("git", "config", "--global", "--unset", key) + return gitSimple("config", "--global", "--unset", key) } // UnsetSystem removes the git config value for the key from the system config func (c *gitConfig) UnsetSystem(key string) (string, error) { - return subprocess.SimpleExec("git", "config", "--system", "--unset", key) + return gitSimple("config", "--system", "--unset", key) } // UnsetGlobalSection removes the entire named section from the global config func (c *gitConfig) UnsetGlobalSection(key string) (string, error) { - return subprocess.SimpleExec("git", "config", "--global", "--remove-section", key) + return gitSimple("config", "--global", "--remove-section", key) } // UnsetSystemSection removes the entire named section from the system config func (c *gitConfig) UnsetSystemSection(key string) (string, error) { - return subprocess.SimpleExec("git", "config", "--system", "--remove-section", key) + return gitSimple("config", "--system", "--remove-section", key) } // UnsetLocalSection removes the entire named section from the system config func (c *gitConfig) UnsetLocalSection(key string) (string, error) { - return subprocess.SimpleExec("git", "config", "--local", "--remove-section", key) + return gitSimple("config", "--local", "--remove-section", key) } // SetLocal sets the git config value for the key in the specified config file @@ -439,7 +544,7 @@ func (c *gitConfig) SetLocal(file, key, val string) (string, error) { args = append(args, "--file", file) } args = append(args, key, val) - return subprocess.SimpleExec("git", args...) + return gitSimple(args...) } // UnsetLocalKey removes the git config value for the key from the specified config file @@ -450,17 +555,17 @@ func (c *gitConfig) UnsetLocalKey(file, key string) (string, error) { args = append(args, "--file", file) } args = append(args, "--unset", key) - return subprocess.SimpleExec("git", args...) + return gitSimple(args...) } // List lists all of the git config values func (c *gitConfig) List() (string, error) { - return subprocess.SimpleExec("git", "config", "-l") + return gitSimple("config", "-l") } // ListFromFile lists all of the git config values in the given config file func (c *gitConfig) ListFromFile(f string) (string, error) { - return subprocess.SimpleExec("git", "config", "-l", "-f", f) + return gitSimple("config", "-l", "-f", f) } // Version returns the git version @@ -469,7 +574,7 @@ func (c *gitConfig) Version() (string, error) { defer c.mu.Unlock() if len(c.gitVersion) == 0 { - v, err := subprocess.SimpleExec("git", "version") + v, err := gitSimple("version") if err != nil { return v, err } @@ -496,7 +601,7 @@ func (c *gitConfig) IsGitVersionAtLeast(ver string) bool { // includeRemoteBranches: true to include refs on remote branches // onlyRemote: set to non-blank to only include remote branches on a single remote func RecentBranches(since time.Time, includeRemoteBranches bool, onlyRemote string) ([]*Ref, error) { - cmd := subprocess.ExecCommand("git", "for-each-ref", + cmd := gitNoLFS("for-each-ref", `--sort=-committerdate`, `--format=%(refname) %(objectname) %(committerdate:iso)`, "refs") @@ -599,7 +704,7 @@ func FormatGitDate(tm time.Time) string { // Get summary information about a commit func GetCommitSummary(commit string) (*CommitSummary, error) { - cmd := subprocess.ExecCommand("git", "show", "-s", + cmd := gitNoLFS("show", "-s", `--format=%H|%h|%P|%ai|%ci|%ae|%an|%ce|%cn|%s`, commit) out, err := cmd.CombinedOutput() @@ -634,7 +739,7 @@ func GetCommitSummary(commit string) (*CommitSummary, error) { } func GitAndRootDirs() (string, string, error) { - cmd := subprocess.ExecCommand("git", "rev-parse", "--git-dir", "--show-toplevel") + cmd := gitNoLFS("rev-parse", "--git-dir", "--show-toplevel") buf := &bytes.Buffer{} cmd.Stderr = buf @@ -669,7 +774,7 @@ func GitAndRootDirs() (string, string, error) { } func RootDir() (string, error) { - cmd := subprocess.ExecCommand("git", "rev-parse", "--show-toplevel") + cmd := gitNoLFS("rev-parse", "--show-toplevel") out, err := cmd.Output() if err != nil { return "", fmt.Errorf("Failed to call git rev-parse --show-toplevel: %v %v", err, string(out)) @@ -685,7 +790,7 @@ func RootDir() (string, error) { } func GitDir() (string, error) { - cmd := subprocess.ExecCommand("git", "rev-parse", "--git-dir") + cmd := gitNoLFS("rev-parse", "--git-dir") out, err := cmd.Output() if err != nil { return "", fmt.Errorf("Failed to call git rev-parse --git-dir: %v %v", err, string(out)) @@ -886,25 +991,7 @@ type CloneFlags struct { // so that files in the working copy will be pointers and not real LFS data func CloneWithoutFilters(flags CloneFlags, args []string) error { - // Before git 2.8, setting filters to blank causes lots of warnings, so use cat instead (slightly slower) - // Also pre 2.2 it failed completely. We used to use it anyway in git 2.2-2.7 and - // suppress the messages in stderr, but doing that with standard StderrPipe suppresses - // the git clone output (git thinks it's not a terminal) and makes it look like it's - // not working. You can get around that with https://github.com/kr/pty but that - // causes difficult issues with passing through Stdin for login prompts - // This way is simpler & more practical. - filterOverride := "" - if !Config.IsGitVersionAtLeast("2.8.0") { - filterOverride = "cat" - } - // Disable the LFS filters while cloning to speed things up - // this is especially effective on Windows where even calling git-lfs at all - // with --skip-smudge is costly across many files in a checkout - cmdargs := []string{ - "-c", fmt.Sprintf("filter.lfs.smudge=%v", filterOverride), - "-c", "filter.lfs.process=", - "-c", "filter.lfs.required=false", - "clone"} + cmdargs := []string{"clone"} // flags if flags.Bare { @@ -1000,7 +1087,7 @@ func CloneWithoutFilters(flags CloneFlags, args []string) error { // Now args cmdargs = append(cmdargs, args...) - cmd := subprocess.ExecCommand("git", cmdargs...) + cmd := gitNoLFS(cmdargs...) // Assign all streams direct cmd.Stdout = os.Stdout @@ -1039,7 +1126,7 @@ func Checkout(treeish string, paths []string, force bool) error { args = append(args, append([]string{"--"}, paths...)...) } - _, err := subprocess.SimpleExec("git", args...) + _, err := gitNoLFSSimple(args...) return err } @@ -1047,7 +1134,7 @@ func Checkout(treeish string, paths []string, force bool) error { // currently cached locally. No remote request is made to verify them. func CachedRemoteRefs(remoteName string) ([]*Ref, error) { var ret []*Ref - cmd := subprocess.ExecCommand("git", "show-ref") + cmd := gitNoLFS("show-ref") outp, err := cmd.StdoutPipe() if err != nil { @@ -1076,7 +1163,7 @@ func CachedRemoteRefs(remoteName string) ([]*Ref, error) { // accessing the remote vir git ls-remote func RemoteRefs(remoteName string) ([]*Ref, error) { var ret []*Ref - cmd := subprocess.ExecCommand("git", "ls-remote", "--heads", "--tags", "-q", remoteName) + cmd := gitNoLFS("ls-remote", "--heads", "--tags", "-q", remoteName) outp, err := cmd.StdoutPipe() if err != nil { @@ -1115,7 +1202,7 @@ func AllRefs() ([]*Ref, error) { // the given working directory "wd", or an error if those references could not // be loaded. func AllRefsIn(wd string) ([]*Ref, error) { - cmd := subprocess.ExecCommand("git", + cmd := gitNoLFS( "for-each-ref", "--format=%(objectname)%00%(refname)") cmd.Dir = wd @@ -1161,7 +1248,7 @@ func GetTrackedFiles(pattern string) ([]string, error) { rootWildcard := len(safePattern) < len(pattern) && strings.ContainsRune(safePattern, '*') var ret []string - cmd := subprocess.ExecCommand("git", + cmd := gitNoLFS( "-c", "core.quotepath=false", // handle special chars in filenames "ls-files", "--cached", // include things which are staged but not committed right now @@ -1220,7 +1307,7 @@ func GetFilesChanged(from, to string) ([]string, error) { } args = append(args, "--") // no ambiguous patterns - cmd := subprocess.ExecCommand("git", args...) + cmd := gitNoLFS(args...) outp, err := cmd.StdoutPipe() if err != nil { return nil, fmt.Errorf("Failed to call git diff: %v", err) @@ -1251,7 +1338,7 @@ func IsFileModified(filepath string) (bool, error) { "--", // separator in case filename ambiguous filepath, } - cmd := subprocess.ExecCommand("git", args...) + cmd := git(args...) outp, err := cmd.StdoutPipe() if err != nil { return false, lfserrors.Wrap(err, "Failed to call git status") diff --git a/git/object_scanner.go b/git/object_scanner.go index 5922e070..bc1ef029 100644 --- a/git/object_scanner.go +++ b/git/object_scanner.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "io/ioutil" - "os/exec" "strconv" "github.com/git-lfs/git-lfs/errors" @@ -53,7 +52,7 @@ type ObjectScanner struct { // // Otherwise, an `*ObjectScanner` is returned with no error. func NewObjectScanner() (*ObjectScanner, error) { - cmd := exec.Command("git", "cat-file", "--batch") + cmd := gitNoLFS("cat-file", "--batch") stdout, err := cmd.StdoutPipe() if err != nil { return nil, errors.Wrap(err, "open stdout") diff --git a/git/rev_list_scanner.go b/git/rev_list_scanner.go index d225e4bf..294dbe5a 100644 --- a/git/rev_list_scanner.go +++ b/git/rev_list_scanner.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "io/ioutil" - "os/exec" "regexp" "strings" "sync" @@ -173,7 +172,7 @@ func NewRevListScanner(include, excluded []string, opt *ScanRefsOptions) (*RevLi return nil, err } - cmd := exec.Command("git", args...) + cmd := gitNoLFS(args...).Cmd if len(opt.WorkingDir) > 0 { cmd.Dir = opt.WorkingDir } diff --git a/lfs/diff_index_scanner.go b/lfs/diff_index_scanner.go index 0fa34d96..c1472d8e 100644 --- a/lfs/diff_index_scanner.go +++ b/lfs/diff_index_scanner.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/git-lfs/git-lfs/errors" + "github.com/git-lfs/git-lfs/git" ) // Status represents the status of a file that appears in the output of `git @@ -122,32 +123,15 @@ type DiffIndexScanner struct { // that error will be returned immediately. Otherwise, a `*DiffIndexScanner` // will be returned with a `nil` error. func NewDiffIndexScanner(ref string, cached bool) (*DiffIndexScanner, error) { - cmd, err := startCommand("git", diffIndexCmdArgs(ref, cached)...) + scanner, err := git.DiffIndex(ref, cached) if err != nil { return nil, err } - - if err = cmd.Stdin.Close(); err != nil { - return nil, err - } - return &DiffIndexScanner{ - from: bufio.NewScanner(cmd.Stdout), + from: scanner, }, nil } -// diffIndexCmdArgs returns a string slice containing the arguments necessary -// to run the diff-index command. -func diffIndexCmdArgs(ref string, cached bool) []string { - args := []string{"diff-index", "-M"} - if cached { - args = append(args, "--cached") - } - args = append(args, ref) - - return args -} - // Scan advances the scan line and yields either a new value for Entry(), or an // Err(). It returns true or false, whether or not it can continue scanning for // more entries. diff --git a/lfs/gitscanner_catfilebatchcheck.go b/lfs/gitscanner_catfilebatchcheck.go index 7790f668..088e0452 100644 --- a/lfs/gitscanner_catfilebatchcheck.go +++ b/lfs/gitscanner_catfilebatchcheck.go @@ -5,6 +5,8 @@ import ( "fmt" "io/ioutil" "strconv" + + "github.com/git-lfs/git-lfs/git" ) // runCatFileBatchCheck uses 'git cat-file --batch-check' to get the type and @@ -13,7 +15,7 @@ import ( // over which strings containing git sha1s will be sent. It returns a channel // from which sha1 strings can be read. func runCatFileBatchCheck(smallRevCh chan string, lockableCh chan string, lockableSet *lockableNameSet, revs *StringChannelWrapper, errCh chan error) error { - cmd, err := startCommand("git", "cat-file", "--batch-check") + cmd, err := git.CatFile() if err != nil { return err } diff --git a/lfs/gitscanner_cmd.go b/lfs/gitscanner_cmd.go deleted file mode 100644 index 355fa567..00000000 --- a/lfs/gitscanner_cmd.go +++ /dev/null @@ -1,49 +0,0 @@ -package lfs - -import ( - "bufio" - "io" - "os/exec" - "strings" - - "github.com/rubyist/tracerx" -) - -type wrappedCmd struct { - Stdin io.WriteCloser - Stdout *bufio.Reader - Stderr *bufio.Reader - *exec.Cmd -} - -// startCommand starts up a command and creates a stdin pipe and a buffered -// stdout & stderr pipes, wrapped in a wrappedCmd. The stdout buffer will be of stdoutBufSize -// bytes. -func startCommand(command string, args ...string) (*wrappedCmd, error) { - cmd := exec.Command(command, args...) - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - - stdin, err := cmd.StdinPipe() - if err != nil { - return nil, err - } - - tracerx.Printf("run_command: %s %s", command, strings.Join(args, " ")) - if err := cmd.Start(); err != nil { - return nil, err - } - - return &wrappedCmd{ - stdin, - bufio.NewReaderSize(stdout, stdoutBufSize), - bufio.NewReaderSize(stderr, stdoutBufSize), - cmd, - }, nil -} diff --git a/lfs/gitscanner_log.go b/lfs/gitscanner_log.go index b75ab4fc..a3ce182d 100644 --- a/lfs/gitscanner_log.go +++ b/lfs/gitscanner_log.go @@ -11,6 +11,7 @@ import ( "github.com/git-lfs/git-lfs/filepathfilter" "github.com/git-lfs/git-lfs/git" + "github.com/git-lfs/git-lfs/subprocess" "github.com/rubyist/tracerx" ) @@ -40,7 +41,7 @@ type gitscannerResult struct { } func scanUnpushed(cb GitScannerFoundPointer, remote string) error { - logArgs := []string{"log", + logArgs := []string{ "--branches", "--tags", // include all locally referenced commits "--not"} // but exclude everything that comes after @@ -53,7 +54,7 @@ func scanUnpushed(cb GitScannerFoundPointer, remote string) error { // Add standard search args to find lfs references logArgs = append(logArgs, logLfsSearchArgs...) - cmd, err := startCommand("git", logArgs...) + cmd, err := git.Log(logArgs...) if err != nil { return err } @@ -62,7 +63,7 @@ func scanUnpushed(cb GitScannerFoundPointer, remote string) error { return nil } -func parseScannerLogOutput(cb GitScannerFoundPointer, direction LogDiffDirection, cmd *wrappedCmd) { +func parseScannerLogOutput(cb GitScannerFoundPointer, direction LogDiffDirection, cmd *subprocess.BufferedCmd) { ch := make(chan gitscannerResult, chanBufSize) go func() { @@ -89,7 +90,7 @@ func parseScannerLogOutput(cb GitScannerFoundPointer, direction LogDiffDirection // logPreviousVersions scans history for all previous versions of LFS pointers // from 'since' up to (but not including) the final state at ref func logPreviousSHAs(cb GitScannerFoundPointer, ref string, since time.Time) error { - logArgs := []string{"log", + logArgs := []string{ fmt.Sprintf("--since=%v", git.FormatGitDate(since)), } // Add standard search args to find lfs references @@ -97,7 +98,7 @@ func logPreviousSHAs(cb GitScannerFoundPointer, ref string, since time.Time) err // ending at ref logArgs = append(logArgs, ref) - cmd, err := startCommand("git", logArgs...) + cmd, err := git.Log(logArgs...) if err != nil { return err } diff --git a/lfs/gitscanner_tree.go b/lfs/gitscanner_tree.go index 83056871..63a4cb46 100644 --- a/lfs/gitscanner_tree.go +++ b/lfs/gitscanner_tree.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/git-lfs/git-lfs/filepathfilter" + "github.com/git-lfs/git-lfs/git" ) // An entry from ls-tree or rev-list including a blob sha and tree path @@ -94,14 +95,7 @@ func catFileBatchTree(treeblobs *TreeBlobChannelWrapper) (*PointerChannelWrapper // The returned channel will be sent these blobs which should be sent to catFileBatchTree // for final check & conversion to Pointer func lsTreeBlobs(ref string, filter *filepathfilter.Filter) (*TreeBlobChannelWrapper, error) { - cmd, err := startCommand("git", "ls-tree", - "-r", // recurse - "-l", // report object size (we'll need this) - "-z", // null line termination - "--full-tree", // start at the root regardless of where we are in it - ref, - ) - + cmd, err := git.LsTree(ref) if err != nil { return nil, err } diff --git a/subprocess/buffered_cmd.go b/subprocess/buffered_cmd.go new file mode 100644 index 00000000..b197c22e --- /dev/null +++ b/subprocess/buffered_cmd.go @@ -0,0 +1,19 @@ +package subprocess + +import ( + "bufio" + "io" +) + +const ( + // stdoutBufSize is the size of the buffers given to a sub-process stdout + stdoutBufSize = 16384 +) + +type BufferedCmd struct { + *Cmd + + Stdin io.WriteCloser + Stdout *bufio.Reader + Stderr *bufio.Reader +} diff --git a/subprocess/subprocess.go b/subprocess/subprocess.go index 00ffe3c7..6bf2e51a 100644 --- a/subprocess/subprocess.go +++ b/subprocess/subprocess.go @@ -3,6 +3,7 @@ package subprocess import ( + "bufio" "bytes" "fmt" "os" @@ -13,6 +14,37 @@ import ( "github.com/rubyist/tracerx" ) +// BufferedExec starts up a command and creates a stdin pipe and a buffered +// stdout & stderr pipes, wrapped in a BufferedCmd. The stdout buffer will be +// of stdoutBufSize bytes. +func BufferedExec(name string, args ...string) (*BufferedCmd, error) { + cmd := ExecCommand(name, args...) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + + if err := cmd.Start(); err != nil { + return nil, err + } + + return &BufferedCmd{ + cmd, + stdin, + bufio.NewReaderSize(stdout, stdoutBufSize), + bufio.NewReaderSize(stderr, stdoutBufSize), + }, nil +} + // SimpleExec is a small wrapper around os/exec.Command. func SimpleExec(name string, args ...string) (string, error) { tracerx.Printf("run_command: '%s' %s", name, strings.Join(args, " "))