lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge

Browse Source
This commit is contained in:
risk danger olson 2016-02-22 11:06:37 -07:00
commit 66d3014776
5 changed files with 142 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/README.md
View File

@ -16,5 +16,5 @@ Quick intro to Git LFS.
## Developer Docs
Details of how the Git LFS client work are in the [official specification](spec.md).
Details of how the Git LFS client works are in the [official specification](spec.md).
There is also an [API specification](api) that describes how the server works.

92
lfs/scanner.go
View File

@ -293,6 +293,8 @@ func revListShas(refLeft, refRight string, opt *ScanRefsOptions) (chan string, e
}
revs <- sha1
}
cmd.Wait()
close(revs)
}()
@ -343,6 +345,8 @@ func revListIndex(cache bool, indexMap *indexFileMap) (chan string, error) {
revs <- sha1
}
}
cmd.Wait()
close(revs)
}()
@ -390,6 +394,7 @@ func catFileBatchCheck(revs chan string) (chan string, error) {
}
}
cmd.Wait()
close(smallRevs)
}()
@ -447,6 +452,8 @@ func catFileBatch(revs chan string) (chan *WrappedPointer, error) {
break
}
}
cmd.Wait()
close(pointers)
}()
@ -569,8 +576,10 @@ func catFileBatchTree(treeblobs chan TreeBlob) (chan *WrappedPointer, error) {
break
}
}
close(pointers)
cmd.Stdin.Close()
cmd.Wait()
close(pointers)
}()
return pointers, nil
@ -584,6 +593,7 @@ func lsTreeBlobs(ref string) (chan TreeBlob, error) {
lsArgs := []string{"ls-tree",
"-r", // recurse
"-l", // report object size (we'll need this)
"-z", // null line termination
"--full-tree", // start at the root regardless of where we are in it
ref}
@ -597,29 +607,65 @@ func lsTreeBlobs(ref string) (chan TreeBlob, error) {
blobs := make(chan TreeBlob, chanBufSize)
go func() {
scanner := bufio.NewScanner(cmd.Stdout)
regex := regexp.MustCompile(`^\d+\s+blob\s+([0-9a-zA-Z]{40})\s+(\d+)\s+(.*)$`)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if match := regex.FindStringSubmatch(line); match != nil {
sz, err := strconv.ParseInt(match[2], 10, 64)
if err != nil {
continue
}
sha1 := match[1]
filename := match[3]
if sz < blobSizeCutoff {
blobs <- TreeBlob{sha1, filename}
}
}
}
parseLsTree(cmd.Stdout, blobs)
cmd.Wait()
close(blobs)
}()
return blobs, nil
}
func parseLsTree(reader io.Reader, output chan TreeBlob) {
scanner := bufio.NewScanner(reader)
scanner.Split(scanNullLines)
for scanner.Scan() {
line := scanner.Text()
parts := strings.SplitN(line, "\t", 2)
if len(parts) < 2 {
continue
}
attrs := strings.SplitN(parts[0], " ", 4)
if len(attrs) < 4 {
continue
}
if attrs[1] != "blob" {
continue
}
sz, err := strconv.ParseInt(strings.TrimSpace(attrs[3]), 10, 64)
if err != nil {
continue
}
if sz < blobSizeCutoff {
sha1 := attrs[2]
filename := parts[1]
output <- TreeBlob{sha1, filename}
}
}
}
func scanNullLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
if i := bytes.IndexByte(data, '\000'); i >= 0 {
// We have a full null-terminated line.
return i + 1, data[0:i], nil
}
// If we're at EOF, we have a final, non-terminated line. Return it.
if atEOF {
return len(data), data, nil
}
// Request more data.
return 0, nil, nil
}
// ScanUnpushed scans history for all LFS pointers which have been added but not
// pushed to the named remote. remoteName can be left blank to mean 'any remote'
func ScanUnpushed(remoteName string) ([]*WrappedPointer, error) {
@ -693,7 +739,10 @@ func ScanUnpushedToChan(remoteName string) (chan *WrappedPointer, error) {
pchan := make(chan *WrappedPointer, chanBufSize)
go parseLogOutputToPointers(cmd.Stdout, LogDiffAdditions, nil, nil, pchan)
go func() {
parseLogOutputToPointers(cmd.Stdout, LogDiffAdditions, nil, nil, pchan)
cmd.Wait()
}()
return pchan, nil
@ -722,7 +771,10 @@ func logPreviousSHAs(ref string, since time.Time) (chan *WrappedPointer, error)
// we pull out deletions, since we want the previous SHAs at commits in the range
// this means we pick up all previous versions that could have been checked
// out in the date range, not just if the commit which *introduced* them is in the range
go parseLogOutputToPointers(cmd.Stdout, LogDiffDeletions, nil, nil, pchan)
go func() {
parseLogOutputToPointers(cmd.Stdout, LogDiffDeletions, nil, nil, pchan)
cmd.Wait()
}()
return pchan, nil

28
lfs/scanner_test.go
View File

@ -228,3 +228,31 @@ func TestParseLogOutputToPointersDeletion(t *testing.T) {
assert.Equal(t, int64(16849), pointers[2].Size)
}
func TestLsTreeParser(t *testing.T) {
stdout := "100644 blob d899f6551a51cf19763c5955c7a06a2726f018e9 42 .gitattributes\000100644 blob 4d343e022e11a8618db494dc3c501e80c7e18197 126 PB SCN 16 Odhrán.wav"
blobs := make(chan TreeBlob, 2)
parseLsTree(strings.NewReader(stdout), blobs)
close(blobs)
<-blobs // gitattributes
blob := <-blobs
if blob.Sha1 != "4d343e022e11a8618db494dc3c501e80c7e18197" {
t.Errorf("Bad sha1: %q", blob.Sha1)
}
if blob.Filename != "PB SCN 16 Odhrán.wav" {
t.Errorf("Bad name: %q", blob.Filename)
}
}
func BenchmarkLsTreeParser(b *testing.B) {
stdout := "100644 blob d899f6551a51cf19763c5955c7a06a2726f018e9 42 .gitattributes\000100644 blob 4d343e022e11a8618db494dc3c501e80c7e18197 126 PB SCN 16 Odhrán.wav"
blobs := make(chan TreeBlob, b.N*2)
// run the Fib function b.N times
for n := 0; n < b.N; n++ {
parseLsTree(strings.NewReader(stdout), blobs)
}
close(blobs)
}

52
test/test-pull.sh
View File

@ -18,58 +18,76 @@ begin_test "pull"
contents="a"
contents_oid=$(calc_oid "$contents")
contents2="A"
contents2_oid=$(calc_oid "$contents2")
printf "$contents" > a.dat
git add a.dat
git add .gitattributes
git commit -m "add a.dat" 2>&1 | tee commit.log
printf "$contents2" > á.dat
git add a.dat á.dat .gitattributes
git commit -m "add files" 2>&1 | tee commit.log
grep "master (root-commit)" commit.log
grep "2 files changed" commit.log
grep "3 files changed" commit.log
grep "create mode 100644 a.dat" commit.log
grep "create mode 100644 .gitattributes" commit.log
ls -al
[ "a" = "$(cat a.dat)" ]
[ "A" = "$(cat "á.dat")" ]
assert_pointer "master" "a.dat" "$contents_oid" 1
assert_pointer "master" "á.dat" "$contents2_oid" 1
refute_server_object "$reponame" "$contents_oid"
refute_server_object "$reponame" "$contents2_oid"
echo "initial push"
git push origin master 2>&1 | tee push.log
grep "(1 of 1 files)" push.log
grep "(2 of 2 files)" push.log
grep "master -> master" push.log
assert_server_object "$reponame" "$contents_oid"
assert_server_object "$reponame" "$contents2_oid"
# change to the clone's working directory
cd ../clone
git pull 2>&1 | grep "Downloading a.dat (1 B)"
echo "normal pull"
git pull 2>&1 | tee pull.log
grep "Downloading a.dat (1 B)" pull.log
grep "Downloading á.dat (1 B)" pull.log
[ "a" = "$(cat a.dat)" ]
[ "A" = "$(cat "á.dat")" ]
assert_local_object "$contents_oid" 1
assert_local_object "$contents2_oid" 1
# Remove the working directory and lfs files
rm a.dat
echo "lfs pull"
rm a.dat á.dat
rm -rf .git/lfs/objects
git lfs pull 2>&1 | grep "(1 of 1 files)"
git lfs pull 2>&1 | grep "(2 of 2 files)"
ls -al
[ "a" = "$(cat a.dat)" ]
[ "A" = "$(cat "á.dat")" ]
assert_local_object "$contents_oid" 1
assert_local_object "$contents2_oid" 1
# Try with remote arg
rm a.dat
echo "lfs pull with remote"
rm a.dat á.dat
rm -rf .git/lfs/objects
git lfs pull origin 2>&1 | grep "(1 of 1 files)"
git lfs pull origin 2>&1 | grep "(2 of 2 files)"
[ "a" = "$(cat a.dat)" ]
[ "A" = "$(cat "á.dat")" ]
assert_local_object "$contents_oid" 1
assert_local_object "$contents2_oid" 1
# Remove just the working directory
rm a.dat
echo "lfs pull with local storage"
rm a.dat á.dat
git lfs pull
[ "a" = "$(cat a.dat)" ]
[ "A" = "$(cat "á.dat")" ]
# Test include / exclude filters supplied in gitconfig
echo "lfs pull with include/exclude filters in gitconfig"
rm -rf .git/lfs/objects
git config "lfs.fetchinclude" "a*"
git lfs pull
@ -81,7 +99,7 @@ begin_test "pull"
git lfs pull
refute_local_object "$contents_oid"
# Test include / exclude filters supplied on the command line
echo "lfs pull with include/exclude filters in command line"
git config --unset "lfs.fetchexclude"
rm -rf .git/lfs/objects
git lfs pull --include="a*"

8
test/testhelpers.sh
View File

@ -10,8 +10,12 @@ assert_pointer() {
local oid="$3"
local size="$4"
tree=$(git ls-tree -lr "$ref")
gitblob=$(echo "$tree" | grep "$path" | cut -f 3 -d " ")
gitblob=$(git ls-tree -lrz "$ref" |
while read -r -d $'\0' x; do
echo $x
done |
grep "$path" | cut -f 3 -d " ")
actual=$(git cat-file -p $gitblob)
expected=$(pointer $oid $size)