update runCatFileBatchCheck() too
This commit is contained in:
parent
5dc3c9c70e
commit
558da776d9
@ -27,7 +27,6 @@ func runCatFileBatch(pointerCh chan *WrappedPointer, revs *StringChannelWrapper,
|
|||||||
scanner := &catFileBatchScanner{r: cmd.Stdout}
|
scanner := &catFileBatchScanner{r: cmd.Stdout}
|
||||||
for r := range revs.Results {
|
for r := range revs.Results {
|
||||||
cmd.Stdin.Write([]byte(r + "\n"))
|
cmd.Stdin.Write([]byte(r + "\n"))
|
||||||
|
|
||||||
p, hasNext, err := scanner.Next()
|
p, hasNext, err := scanner.Next()
|
||||||
if p != nil {
|
if p != nil {
|
||||||
pointerCh <- p
|
pointerCh <- p
|
||||||
@ -68,7 +67,7 @@ type catFileBatchScanner struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *catFileBatchScanner) Next() (*WrappedPointer, bool, error) {
|
func (s *catFileBatchScanner) Next() (*WrappedPointer, bool, error) {
|
||||||
p, err := s.scanChunk()
|
p, err := s.next()
|
||||||
switch err {
|
switch err {
|
||||||
case nil:
|
case nil:
|
||||||
return p, true, nil
|
return p, true, nil
|
||||||
@ -79,7 +78,7 @@ func (s *catFileBatchScanner) Next() (*WrappedPointer, bool, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *catFileBatchScanner) scanChunk() (*WrappedPointer, error) {
|
func (s *catFileBatchScanner) next() (*WrappedPointer, error) {
|
||||||
l, err := s.r.ReadBytes('\n')
|
l, err := s.r.ReadBytes('\n')
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -3,7 +3,6 @@ package lfs
|
|||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"strconv"
|
"strconv"
|
||||||
)
|
)
|
||||||
@ -19,43 +18,41 @@ func runCatFileBatchCheck(smallRevCh chan string, revs *StringChannelWrapper, er
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
go catFileBatchCheckOutput(smallRevCh, cmd, errCh)
|
go func() {
|
||||||
go catFileBatchCheckInput(cmd, revs, errCh)
|
scanner := &catFileBatchCheckScanner{s: bufio.NewScanner(cmd.Stdout), limit: blobSizeCutoff}
|
||||||
|
for r := range revs.Results {
|
||||||
|
cmd.Stdin.Write([]byte(r + "\n"))
|
||||||
|
blobOid, hasNext, err := scanner.Next()
|
||||||
|
if len(blobOid) > 0 {
|
||||||
|
smallRevCh <- blobOid
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
errCh <- err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !hasNext {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := revs.Wait(); err != nil {
|
||||||
|
errCh <- err
|
||||||
|
}
|
||||||
|
cmd.Stdin.Close()
|
||||||
|
|
||||||
|
stderr, _ := ioutil.ReadAll(cmd.Stderr)
|
||||||
|
err := cmd.Wait()
|
||||||
|
if err != nil {
|
||||||
|
errCh <- fmt.Errorf("Error in git cat-file --batch-check: %v %v", err, string(stderr))
|
||||||
|
}
|
||||||
|
close(smallRevCh)
|
||||||
|
close(errCh)
|
||||||
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func catFileBatchCheckOutput(smallRevCh chan string, cmd *wrappedCmd, errCh chan error) {
|
|
||||||
scanner := &catFileBatchCheckScanner{s: bufio.NewScanner(cmd.Stdout), limit: blobSizeCutoff}
|
|
||||||
for scanner.Scan() {
|
|
||||||
smallRevCh <- scanner.BlobOID()
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := scanner.Err(); err != nil {
|
|
||||||
errCh <- err
|
|
||||||
}
|
|
||||||
|
|
||||||
stderr, _ := ioutil.ReadAll(cmd.Stderr)
|
|
||||||
err := cmd.Wait()
|
|
||||||
if err != nil {
|
|
||||||
errCh <- fmt.Errorf("Error in git cat-file --batch-check: %v %v", err, string(stderr))
|
|
||||||
}
|
|
||||||
close(smallRevCh)
|
|
||||||
close(errCh)
|
|
||||||
}
|
|
||||||
|
|
||||||
func catFileBatchCheckInput(cmd *wrappedCmd, revs *StringChannelWrapper, errCh chan error) {
|
|
||||||
for r := range revs.Results {
|
|
||||||
cmd.Stdin.Write([]byte(r + "\n"))
|
|
||||||
}
|
|
||||||
err := revs.Wait()
|
|
||||||
if err != nil {
|
|
||||||
// We can share errchan with other goroutine since that won't close it
|
|
||||||
// until we close the stdin below
|
|
||||||
errCh <- err
|
|
||||||
}
|
|
||||||
cmd.Stdin.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
type catFileBatchCheckScanner struct {
|
type catFileBatchCheckScanner struct {
|
||||||
s *bufio.Scanner
|
s *bufio.Scanner
|
||||||
limit int
|
limit int
|
||||||
@ -63,56 +60,31 @@ type catFileBatchCheckScanner struct {
|
|||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *catFileBatchCheckScanner) BlobOID() string {
|
func (s *catFileBatchCheckScanner) Next() (string, bool, error) {
|
||||||
return s.blobOID
|
hasNext := s.s.Scan()
|
||||||
}
|
line := s.s.Text()
|
||||||
|
lineLen := len(line)
|
||||||
|
|
||||||
func (s *catFileBatchCheckScanner) Err() error {
|
// Format is:
|
||||||
return s.err
|
// <sha1> <type> <size>
|
||||||
}
|
// type is at a fixed spot, if we see that it's "blob", we can avoid
|
||||||
|
// splitting the line just to get the size.
|
||||||
|
if lineLen < 46 {
|
||||||
|
return "", hasNext, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *catFileBatchCheckScanner) Scan() bool {
|
if line[41:45] != "blob" {
|
||||||
s.blobOID, s.err = "", nil
|
return "", hasNext, nil
|
||||||
b, err := scanBlobOID(s.s, s.limit)
|
}
|
||||||
|
|
||||||
|
size, err := strconv.Atoi(line[46:lineLen])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// EOF halts scanning, but isn't a reportable error
|
return "", hasNext, nil
|
||||||
if err != io.EOF {
|
|
||||||
s.err = err
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
s.blobOID = b
|
if size >= s.limit {
|
||||||
return true
|
return "", hasNext, nil
|
||||||
}
|
|
||||||
|
|
||||||
func scanBlobOID(s *bufio.Scanner, limit int) (string, error) {
|
|
||||||
objType := "blob"
|
|
||||||
for s.Scan() {
|
|
||||||
line := s.Text()
|
|
||||||
lineLen := len(line)
|
|
||||||
|
|
||||||
// Format is:
|
|
||||||
// <sha1> <type> <size>
|
|
||||||
// type is at a fixed spot, if we see that it's "blob", we can avoid
|
|
||||||
// splitting the line just to get the size.
|
|
||||||
if lineLen < 46 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if line[41:45] != objType {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
size, err := strconv.Atoi(line[46:lineLen])
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if size < limit {
|
|
||||||
return line[0:40], nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return "", io.EOF
|
return line[0:40], hasNext, nil
|
||||||
}
|
}
|
||||||
|
@ -23,12 +23,36 @@ func TestCatFileBatchCheckScannerWithValidOutput(t *testing.T) {
|
|||||||
limit: 1024,
|
limit: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
var found []string
|
assertNextEmptyString(t, s)
|
||||||
for s.Scan() {
|
assertNextEmptyString(t, s)
|
||||||
found = append(found, s.BlobOID())
|
assertNextEmptyString(t, s)
|
||||||
}
|
assertNextOID(t, s, "0000000000000000000000000000000000000002")
|
||||||
|
assertNextEmptyString(t, s)
|
||||||
assert.Nil(t, s.Err())
|
assertNextEmptyString(t, s)
|
||||||
assert.Equal(t, 1, len(found))
|
assertStringScannerDone(t, s)
|
||||||
assert.Equal(t, "0000000000000000000000000000000000000002", found[0])
|
}
|
||||||
|
|
||||||
|
type stringScanner interface {
|
||||||
|
Next() (string, bool, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertNextOID(t *testing.T, scanner stringScanner, oid string) {
|
||||||
|
actual, hasNext, err := scanner.Next()
|
||||||
|
assert.Equal(t, oid, actual)
|
||||||
|
assert.Nil(t, err)
|
||||||
|
assert.True(t, hasNext)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertNextEmptyString(t *testing.T, scanner stringScanner) {
|
||||||
|
actual, hasNext, err := scanner.Next()
|
||||||
|
assert.Equal(t, "", actual)
|
||||||
|
assert.Nil(t, err)
|
||||||
|
assert.True(t, hasNext)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertStringScannerDone(t *testing.T, scanner stringScanner) {
|
||||||
|
actual, hasNext, err := scanner.Next()
|
||||||
|
assert.Equal(t, "", actual)
|
||||||
|
assert.Nil(t, err)
|
||||||
|
assert.False(t, hasNext)
|
||||||
}
|
}
|
||||||
|
@ -35,22 +35,22 @@ func TestCatFileBatchScannerWithValidOutput(t *testing.T) {
|
|||||||
scanner := &catFileBatchScanner{r: bufio.NewReader(reader)}
|
scanner := &catFileBatchScanner{r: bufio.NewReader(reader)}
|
||||||
|
|
||||||
for i := 0; i < 5; i++ {
|
for i := 0; i < 5; i++ {
|
||||||
assertNextEmpty(t, scanner)
|
assertNextEmptyPointer(t, scanner)
|
||||||
}
|
}
|
||||||
|
|
||||||
assertNextPointer(t, scanner, "e71eefd918ea175b8f362611f981f648dbf9888ff74865077cb4c9077728f350")
|
assertNextPointer(t, scanner, "e71eefd918ea175b8f362611f981f648dbf9888ff74865077cb4c9077728f350")
|
||||||
|
|
||||||
for i := 0; i < 5; i++ {
|
for i := 0; i < 5; i++ {
|
||||||
assertNextEmpty(t, scanner)
|
assertNextEmptyPointer(t, scanner)
|
||||||
}
|
}
|
||||||
|
|
||||||
assertNextPointer(t, scanner, "0eb69b651be65d5a61d6bebf2c53c811a5bf8031951111000e2077f4d7fe43b1")
|
assertNextPointer(t, scanner, "0eb69b651be65d5a61d6bebf2c53c811a5bf8031951111000e2077f4d7fe43b1")
|
||||||
|
|
||||||
for i := 0; i < 5; i++ {
|
for i := 0; i < 5; i++ {
|
||||||
assertNextEmpty(t, scanner)
|
assertNextEmptyPointer(t, scanner)
|
||||||
}
|
}
|
||||||
|
|
||||||
assertScannerDone(t, scanner)
|
assertPointerScannerDone(t, scanner)
|
||||||
}
|
}
|
||||||
|
|
||||||
type pointerScanner interface {
|
type pointerScanner interface {
|
||||||
@ -65,14 +65,14 @@ func assertNextPointer(t *testing.T, scanner pointerScanner, oid string) {
|
|||||||
assert.True(t, hasNext)
|
assert.True(t, hasNext)
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertNextEmpty(t *testing.T, scanner pointerScanner) {
|
func assertNextEmptyPointer(t *testing.T, scanner pointerScanner) {
|
||||||
p, hasNext, err := scanner.Next()
|
p, hasNext, err := scanner.Next()
|
||||||
assert.Nil(t, err)
|
assert.Nil(t, err)
|
||||||
assert.Nil(t, p)
|
assert.Nil(t, p)
|
||||||
assert.True(t, hasNext)
|
assert.True(t, hasNext)
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertScannerDone(t *testing.T, scanner pointerScanner) {
|
func assertPointerScannerDone(t *testing.T, scanner pointerScanner) {
|
||||||
p, hasNext, err := scanner.Next()
|
p, hasNext, err := scanner.Next()
|
||||||
assert.Nil(t, err)
|
assert.Nil(t, err)
|
||||||
assert.Nil(t, p)
|
assert.Nil(t, p)
|
||||||
|
Loading…
Reference in New Issue
Block a user