lfs/catfilebatch: don't read large blobs into memory
This commit is contained in:
parent
3a858da316
commit
844c0b0db2
@ -162,24 +162,35 @@ func (s *CatFileBatchScanner) next() (string, string, *WrappedPointer, error) {
|
||||
blobSha := string(fields[0])
|
||||
size, _ := strconv.Atoi(string(fields[2]))
|
||||
sha := sha256.New()
|
||||
buf := make([]byte, size)
|
||||
read, err := io.ReadFull(io.TeeReader(s.r, sha), buf)
|
||||
|
||||
var buf *bytes.Buffer
|
||||
var to io.Writer = sha
|
||||
if size <= blobSizeCutoff {
|
||||
buf = bytes.NewBuffer(make([]byte, 0, size))
|
||||
to = io.MultiWriter(to, buf)
|
||||
}
|
||||
|
||||
read, err := io.CopyN(to, s.r, int64(size))
|
||||
if err != nil {
|
||||
return blobSha, "", nil, err
|
||||
}
|
||||
|
||||
if size != read {
|
||||
if int64(size) != read {
|
||||
return blobSha, "", nil, fmt.Errorf("expected %d bytes, read %d bytes", size, read)
|
||||
}
|
||||
|
||||
p, err := DecodePointer(bytes.NewBuffer(buf[:read]))
|
||||
var pointer *WrappedPointer
|
||||
var contentsSha string
|
||||
if err == nil {
|
||||
contentsSha = p.Oid
|
||||
pointer = &WrappedPointer{
|
||||
Sha1: blobSha,
|
||||
Pointer: p,
|
||||
|
||||
if size <= blobSizeCutoff {
|
||||
if p, err := DecodePointer(bytes.NewReader(buf.Bytes())); err != nil {
|
||||
contentsSha = fmt.Sprintf("%x", sha.Sum(nil))
|
||||
} else {
|
||||
pointer = &WrappedPointer{
|
||||
Sha1: blobSha,
|
||||
Pointer: p,
|
||||
}
|
||||
contentsSha = p.Oid
|
||||
}
|
||||
} else {
|
||||
contentsSha = fmt.Sprintf("%x", sha.Sum(nil))
|
||||
|
@ -3,12 +3,14 @@ package lfs
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCatFileBatchScannerWithValidOutput(t *testing.T) {
|
||||
@ -55,6 +57,28 @@ func TestCatFileBatchScannerWithValidOutput(t *testing.T) {
|
||||
assert.Nil(t, scanner.Pointer())
|
||||
}
|
||||
|
||||
func TestCatFileBatchScannerWithLargeBlobs(t *testing.T) {
|
||||
buf := bytes.NewBuffer(make([]byte, 0, 1025))
|
||||
sha := sha256.New()
|
||||
rng := rand.New(rand.NewSource(0))
|
||||
|
||||
_, err := io.CopyN(io.MultiWriter(sha, buf), rng, 1025)
|
||||
require.Nil(t, err)
|
||||
|
||||
fake := bytes.NewBuffer(nil)
|
||||
writeFakeBuffer(t, fake, buf.Bytes(), buf.Len())
|
||||
|
||||
scanner := &CatFileBatchScanner{r: bufio.NewReader(fake)}
|
||||
|
||||
require.True(t, scanner.Scan(nil))
|
||||
assert.Nil(t, scanner.Pointer())
|
||||
assert.Equal(t, fmt.Sprintf("%x", sha.Sum(nil)), scanner.ContentsSha())
|
||||
|
||||
assert.False(t, scanner.Scan(nil))
|
||||
assert.Nil(t, scanner.Err())
|
||||
assert.Nil(t, scanner.Pointer())
|
||||
}
|
||||
|
||||
func assertNextPointer(t *testing.T, scanner *CatFileBatchScanner, oid string) {
|
||||
assert.True(t, scanner.Scan(nil))
|
||||
assert.Nil(t, scanner.Err())
|
||||
|
Loading…
Reference in New Issue
Block a user