2016-11-17 22:54:05 +00:00
|
|
|
package lfs
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"io/ioutil"
|
2016-12-06 23:52:28 +00:00
|
|
|
|
2019-08-09 15:25:23 +00:00
|
|
|
"github.com/git-lfs/git-lfs/config"
|
2016-12-06 23:52:28 +00:00
|
|
|
"github.com/git-lfs/git-lfs/filepathfilter"
|
2017-08-21 10:16:10 +00:00
|
|
|
"github.com/git-lfs/git-lfs/git"
|
2016-11-17 22:54:05 +00:00
|
|
|
)
|
|
|
|
|
2020-05-15 18:58:37 +00:00
|
|
|
func runScanTree(cb GitScannerFoundPointer, ref string, filter *filepathfilter.Filter, gitEnv, osEnv config.Environment) error {
|
2016-11-17 22:54:05 +00:00
|
|
|
// We don't use the nameMap approach here since that's imprecise when >1 file
|
|
|
|
// can be using the same content
|
2021-06-03 18:18:59 +00:00
|
|
|
treeShas, err := lsTreeBlobs(ref, func(t *git.TreeBlob) bool {
|
|
|
|
return t != nil && t.Size < blobSizeCutoff && filter.Allows(t.Filename)
|
|
|
|
})
|
2016-11-17 22:54:05 +00:00
|
|
|
if err != nil {
|
2016-11-29 17:56:03 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2020-05-15 18:58:37 +00:00
|
|
|
pcw, err := catFileBatchTree(treeShas, gitEnv, osEnv)
|
2016-11-29 17:56:03 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
2016-11-17 22:54:05 +00:00
|
|
|
}
|
|
|
|
|
2016-11-29 17:56:03 +00:00
|
|
|
for p := range pcw.Results {
|
|
|
|
cb(p, nil)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := pcw.Wait(); err != nil {
|
|
|
|
cb(nil, err)
|
|
|
|
}
|
|
|
|
return nil
|
2016-11-17 22:54:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// catFileBatchTree uses git cat-file --batch to get the object contents
|
|
|
|
// of a git object, given its sha1. The contents will be decoded into
|
|
|
|
// a Git LFS pointer. treeblobs is a channel over which blob entries
|
|
|
|
// will be sent. It returns a channel from which point.Pointers can be read.
|
2020-05-15 18:58:37 +00:00
|
|
|
func catFileBatchTree(treeblobs *TreeBlobChannelWrapper, gitEnv, osEnv config.Environment) (*PointerChannelWrapper, error) {
|
|
|
|
scanner, err := NewPointerScanner(gitEnv, osEnv)
|
2016-11-17 22:54:05 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
pointers := make(chan *WrappedPointer, chanBufSize)
|
|
|
|
errchan := make(chan error, 10) // Multiple errors possible
|
|
|
|
|
|
|
|
go func() {
|
2018-11-08 19:12:03 +00:00
|
|
|
hasNext := true
|
2016-11-17 22:54:05 +00:00
|
|
|
for t := range treeblobs.Results {
|
2021-04-21 19:47:25 +00:00
|
|
|
hasNext = scanner.Scan(t.Oid)
|
2018-11-08 19:12:03 +00:00
|
|
|
|
2016-11-19 00:00:57 +00:00
|
|
|
if p := scanner.Pointer(); p != nil {
|
2016-11-18 21:20:51 +00:00
|
|
|
p.Name = t.Filename
|
|
|
|
pointers <- p
|
2016-11-17 22:54:05 +00:00
|
|
|
}
|
2016-11-18 22:26:29 +00:00
|
|
|
|
2016-11-19 00:00:57 +00:00
|
|
|
if err := scanner.Err(); err != nil {
|
2016-11-18 22:26:29 +00:00
|
|
|
errchan <- err
|
|
|
|
}
|
|
|
|
|
|
|
|
if !hasNext {
|
|
|
|
break
|
|
|
|
}
|
2016-11-17 22:54:05 +00:00
|
|
|
}
|
2016-11-18 21:20:51 +00:00
|
|
|
|
2018-11-08 19:12:03 +00:00
|
|
|
// If the scanner quit early, we may still have treeblobs to
|
|
|
|
// read, so waiting for it to close will cause a deadlock.
|
|
|
|
if hasNext {
|
|
|
|
// Deal with nested error from incoming treeblobs
|
|
|
|
err := treeblobs.Wait()
|
|
|
|
if err != nil {
|
|
|
|
errchan <- err
|
|
|
|
}
|
2016-11-17 22:54:05 +00:00
|
|
|
}
|
|
|
|
|
2017-03-22 21:15:02 +00:00
|
|
|
if err = scanner.Close(); err != nil {
|
|
|
|
errchan <- err
|
2016-11-17 22:54:05 +00:00
|
|
|
}
|
2017-03-22 21:15:02 +00:00
|
|
|
|
2016-11-17 22:54:05 +00:00
|
|
|
close(pointers)
|
|
|
|
close(errchan)
|
|
|
|
}()
|
|
|
|
|
|
|
|
return NewPointerChannelWrapper(pointers, errchan), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Use ls-tree at ref to find a list of candidate tree blobs which might be lfs files
|
|
|
|
// The returned channel will be sent these blobs which should be sent to catFileBatchTree
|
|
|
|
// for final check & conversion to Pointer
|
2021-06-03 18:18:59 +00:00
|
|
|
func lsTreeBlobs(ref string, predicate func(*git.TreeBlob) bool) (*TreeBlobChannelWrapper, error) {
|
2017-08-21 10:16:10 +00:00
|
|
|
cmd, err := git.LsTree(ref)
|
2016-11-17 22:54:05 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
cmd.Stdin.Close()
|
|
|
|
|
2021-04-21 19:47:25 +00:00
|
|
|
blobs := make(chan git.TreeBlob, chanBufSize)
|
2016-11-17 22:54:05 +00:00
|
|
|
errchan := make(chan error, 1)
|
|
|
|
|
|
|
|
go func() {
|
2021-04-21 19:47:25 +00:00
|
|
|
scanner := git.NewLsTreeScanner(cmd.Stdout)
|
2016-11-19 00:19:45 +00:00
|
|
|
for scanner.Scan() {
|
2021-06-03 18:18:59 +00:00
|
|
|
if t := scanner.TreeBlob(); predicate(t) {
|
2016-11-19 00:19:45 +00:00
|
|
|
blobs <- *t
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-11-17 22:54:05 +00:00
|
|
|
stderr, _ := ioutil.ReadAll(cmd.Stderr)
|
|
|
|
err := cmd.Wait()
|
|
|
|
if err != nil {
|
2019-10-22 07:15:05 +00:00
|
|
|
errchan <- fmt.Errorf("error in git ls-tree: %v %v", err, string(stderr))
|
2016-11-17 22:54:05 +00:00
|
|
|
}
|
|
|
|
close(blobs)
|
|
|
|
close(errchan)
|
|
|
|
}()
|
|
|
|
|
|
|
|
return NewTreeBlobChannelWrapper(blobs, errchan), nil
|
|
|
|
}
|