lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into safe-track-patterns

Browse Source
This commit is contained in:
Taylor Blau 2016-07-05 13:38:22 -06:00
commit 08afb68bc1
8 changed files with 417 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
commands/command_checkout.go
View File

@ -20,7 +20,6 @@ var (
Use: "checkout",
Run: checkoutCommand,
}
checkoutUnstagedArg bool
)
func checkoutCommand(cmd *cobra.Command, args []string) {
@ -44,7 +43,6 @@ func checkoutCommand(cmd *cobra.Command, args []string) {
}
func init() {
checkoutCmd.Flags().BoolVarP(&checkoutUnstagedArg, "unstaged", "u", false, "Do not add files to the index")
RootCmd.AddCommand(checkoutCmd)
}
@ -209,7 +207,7 @@ func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
}
}
if cmd == nil && !checkoutUnstagedArg {
if cmd == nil {
// Fire up the update-index command
cmd = exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
updateIdxStdin, err = cmd.StdinPipe()
@ -223,9 +221,7 @@ func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
}
if updateIdxStdin != nil {
updateIdxStdin.Write([]byte(cwdfilepath + "\n"))
}
updateIdxStdin.Write([]byte(cwdfilepath + "\n"))
}
close(repopathchan)

7
docs/man/git-lfs-checkout.1.ronn
View File

@ -3,7 +3,7 @@ git-lfs-checkout(1) -- Update working copy with file content if available
## SYNOPSIS
`git lfs checkout` [options] <filespec>...
`git lfs checkout` <filespec>...
## DESCRIPTION
@ -18,11 +18,6 @@ we have it in the local store. Modified files are never overwritten.
Filespecs can be provided as arguments to restrict the files which are updated.
## OPTIONS
* `--unstaged` `-u`:
Do not add files to the index, keeping them unstaged.
## EXAMPLES
* Checkout all files that are missing or placeholders

164
test/cmd/lfstest-gitserver.go
View File

@ -317,7 +317,18 @@ func lfsBatchHandler(w http.ResponseWriter, r *http.Request, repo string) {
res := []lfsObject{}
testingChunked := testingChunkedTransferEncoding(r)
testingTus := testingTusUploadInBatchReq(r)
testingTusInterrupt := testingTusUploadInterruptedInBatchReq(r)
var transferChoice string
if testingTus {
for _, t := range objs.Transfers {
if t == "tus" {
transferChoice = "tus"
break
}
}
}
for _, obj := range objs.Objects {
action := objs.Operation
@ -365,6 +376,9 @@ func lfsBatchHandler(w http.ResponseWriter, r *http.Request, repo string) {
if testingChunked {
o.Actions[action].Header["Transfer-Encoding"] = "chunked"
}
if testingTusInterrupt {
o.Actions[action].Header["Lfs-Tus-Interrupt"] = "true"
}
res = append(res, o)
}
@ -385,6 +399,7 @@ func lfsBatchHandler(w http.ResponseWriter, r *http.Request, repo string) {
// Persistent state across requests
var batchResumeFailFallbackStorageAttempts = 0
var tusStorageAttempts = 0
// handles any /storage/{oid} requests
func storageHandler(w http.ResponseWriter, r *http.Request) {
@ -431,6 +446,7 @@ func storageHandler(w http.ResponseWriter, r *http.Request) {
hash := sha256.New()
buf := &bytes.Buffer{}
io.Copy(io.MultiWriter(hash, buf), r.Body)
oid := hex.EncodeToString(hash.Sum(nil))
if !strings.HasSuffix(r.URL.Path, "/"+oid) {
@ -487,11 +503,110 @@ func storageHandler(w http.ResponseWriter, r *http.Request) {
}
w.WriteHeader(404)
case "HEAD":
// tus.io
if !validateTusHeaders(r) {
w.WriteHeader(400)
return
}
parts := strings.Split(r.URL.Path, "/")
oid := parts[len(parts)-1]
var offset int64
if by, ok := largeObjects.GetIncomplete(repo, oid); ok {
offset = int64(len(by))
}
w.Header().Set("Upload-Offset", strconv.FormatInt(offset, 10))
w.WriteHeader(200)
case "PATCH":
// tus.io
if !validateTusHeaders(r) {
w.WriteHeader(400)
return
}
parts := strings.Split(r.URL.Path, "/")
oid := parts[len(parts)-1]
offsetHdr := r.Header.Get("Upload-Offset")
offset, err := strconv.ParseInt(offsetHdr, 10, 64)
if err != nil {
log.Fatal("Unable to parse Upload-Offset header in request: ", err)
w.WriteHeader(400)
return
}
hash := sha256.New()
buf := &bytes.Buffer{}
out := io.MultiWriter(hash, buf)
if by, ok := largeObjects.GetIncomplete(repo, oid); ok {
if offset != int64(len(by)) {
log.Fatal(fmt.Sprintf("Incorrect offset in request, got %d expected %d", offset, len(by)))
w.WriteHeader(400)
return
}
_, err := out.Write(by)
if err != nil {
log.Fatal("Error reading incomplete bytes from store: ", err)
w.WriteHeader(500)
return
}
largeObjects.DeleteIncomplete(repo, oid)
log.Printf("Resuming upload of %v at byte %d", oid, offset)
}
// As a test, we intentionally break the upload from byte 0 by only
// reading some bytes the quitting & erroring, this forces a resume
// any offset > 0 will work ok
var copyErr error
if r.Header.Get("Lfs-Tus-Interrupt") == "true" && offset == 0 {
chdr := r.Header.Get("Content-Length")
contentLen, err := strconv.ParseInt(chdr, 10, 64)
if err != nil {
log.Fatal(fmt.Sprintf("Invalid Content-Length %q", chdr))
w.WriteHeader(400)
return
}
truncated := contentLen / 3
_, _ = io.CopyN(out, r.Body, truncated)
r.Body.Close()
copyErr = fmt.Errorf("Simulated copy error")
} else {
_, copyErr = io.Copy(out, r.Body)
}
if copyErr != nil {
b := buf.Bytes()
if len(b) > 0 {
log.Printf("Incomplete upload of %v, %d bytes", oid, len(b))
largeObjects.SetIncomplete(repo, oid, b)
}
w.WriteHeader(500)
} else {
checkoid := hex.EncodeToString(hash.Sum(nil))
if checkoid != oid {
log.Fatal(fmt.Sprintf("Incorrect oid after calculation, got %q expected %q", checkoid, oid))
w.WriteHeader(403)
return
}
b := buf.Bytes()
largeObjects.Set(repo, oid, b)
w.Header().Set("Upload-Offset", strconv.FormatInt(int64(len(b)), 10))
w.WriteHeader(204)
}
default:
w.WriteHeader(405)
}
}
func validateTusHeaders(r *http.Request) bool {
if len(r.Header.Get("Tus-Resumable")) == 0 {
log.Fatal("Missing Tus-Resumable header in request")
return false
}
return true
}
func gitHandler(w http.ResponseWriter, r *http.Request) {
defer func() {
io.Copy(ioutil.Discard, r.Body)
@ -793,6 +908,13 @@ func testingChunkedTransferEncoding(r *http.Request) bool {
return strings.HasPrefix(r.URL.String(), "/test-chunked-transfer-encoding")
}
func testingTusUploadInBatchReq(r *http.Request) bool {
return strings.HasPrefix(r.URL.String(), "/test-tus-upload")
}
func testingTusUploadInterruptedInBatchReq(r *http.Request) bool {
return strings.HasPrefix(r.URL.String(), "/test-tus-upload-interrupt")
}
var lfsUrlRE = regexp.MustCompile(`\A/?([^/]+)/info/lfs`)
func repoFromLfsUrl(urlpath string) (string, error) {
@ -809,8 +931,9 @@ func repoFromLfsUrl(urlpath string) (string, error) {
}
type lfsStorage struct {
objects map[string]map[string][]byte
mutex *sync.Mutex
objects map[string]map[string][]byte
incomplete map[string]map[string][]byte
mutex *sync.Mutex
}
func (s *lfsStorage) Get(repo, oid string) ([]byte, bool) {
@ -857,10 +980,43 @@ func (s *lfsStorage) Delete(repo, oid string) {
}
}
func (s *lfsStorage) GetIncomplete(repo, oid string) ([]byte, bool) {
s.mutex.Lock()
defer s.mutex.Unlock()
repoObjects, ok := s.incomplete[repo]
if !ok {
return nil, ok
}
by, ok := repoObjects[oid]
return by, ok
}
func (s *lfsStorage) SetIncomplete(repo, oid string, by []byte) {
s.mutex.Lock()
defer s.mutex.Unlock()
repoObjects, ok := s.incomplete[repo]
if !ok {
repoObjects = make(map[string][]byte)
s.incomplete[repo] = repoObjects
}
repoObjects[oid] = by
}
func (s *lfsStorage) DeleteIncomplete(repo, oid string) {
s.mutex.Lock()
defer s.mutex.Unlock()
repoObjects, ok := s.incomplete[repo]
if ok {
delete(repoObjects, oid)
}
}
func newLfsStorage() *lfsStorage {
return &lfsStorage{
objects: make(map[string]map[string][]byte),
mutex: &sync.Mutex{},
objects: make(map[string]map[string][]byte),
incomplete: make(map[string]map[string][]byte),
mutex: &sync.Mutex{},
}
}

31
test/test-checkout.sh
View File

@ -119,34 +119,3 @@ begin_test "checkout: outside git repository"
grep "Not in a git repository" checkout.log
)
end_test
begin_test "checkout --unstaged"
(
set -e
reponame="$(basename "$0" ".sh")-unstaged"
setup_remote_repo "$reponame"
clone_repo "$reponame" repo-unstaged
git lfs track "*.dat" 2>&1 | tee track.log
contents="something something"
printf "$contents" > file1.dat
git add file1.dat
git add .gitattributes
git commit -m "add files"
# Remove the working directory
rm -f file1.dat
echo "checkout should replace all"
git lfs checkout --unstaged
[ "$contents" = "$(cat file1.dat)" ]
echo "large files should not be staged"
git diff-files
git diff-files | grep file1.dat
)
end_test

69
test/test-resume-tus.sh Executable file
View File

@ -0,0 +1,69 @@
#!/usr/bin/env bash
. "test/testlib.sh"
begin_test "tus-upload-uninterrupted"
(
set -e
# this repo name is the indicator to the server to use tus
reponame="test-tus-upload"
setup_remote_repo "$reponame"
clone_repo "$reponame" $reponame
git lfs track "*.dat" 2>&1 | tee track.log
grep "Tracking \*.dat" track.log
contents="jksgdfljkgsdlkjafg lsjdgf alkjgsd lkfjag sldjkgf alkjsgdflkjagsd kljfg asdjgf kalsd"
contents_oid=$(calc_oid "$contents")
printf "$contents" > a.dat
git add a.dat
git add .gitattributes
git commit -m "add a.dat" 2>&1 | tee commit.log
GIT_TRACE=1 git push origin master 2>&1 | tee pushtus.log
grep "xfer: tus.io uploading" pushtus.log
assert_server_object "$reponame" "$contents_oid"
)
end_test
begin_test "tus-upload-interrupted-resume"
(
set -e
# this repo name is the indicator to the server to use tus, AND to
# interrupt the upload part way
reponame="test-tus-upload-interrupt"
setup_remote_repo "$reponame"
clone_repo "$reponame" $reponame
git lfs track "*.dat" 2>&1 | tee track.log
grep "Tracking \*.dat" track.log
# this string announces to server that we want it to abort the download part
# way, but reject the Range: header and fall back on re-downloading instead
contents="234587134187634598o634857619384765b747qcvtuedvoaicwtvseudtvcoqi7280r7qvow4i7r8c46pr9q6v9pri6ioq2r8"
contents_oid=$(calc_oid "$contents")
printf "$contents" > a.dat
git add a.dat
git add .gitattributes
git commit -m "add a.dat" 2>&1 | tee commit.log
GIT_TRACE=1 git push origin master 2>&1 | tee pushtus_resume.log
# first attempt will start from the beginning
grep "xfer: tus.io uploading" pushtus_resume.log
grep "HTTP: 500" pushtus_resume.log
# that will have failed but retry on 500 will resume it
grep "xfer: tus.io resuming" pushtus_resume.log
grep "HTTP: 204" pushtus_resume.log
# should have completed in the end
assert_server_object "$reponame" "$contents_oid"
)
end_test

18
transfer/adapterbase.go
View File

@ -120,3 +120,21 @@ func (a *adapterBase) worker(workerNum int) {
tracerx.Printf("xfer: adapter %q worker %d stopping", a.Name(), workerNum)
a.workerWait.Done()
}
func advanceCallbackProgress(cb TransferProgressCallback, t *Transfer, numBytes int64) {
if cb != nil {
// Must split into max int sizes since read count is int
const maxInt = int(^uint(0) >> 1)
for read := int64(0); read < numBytes; {
remainder := numBytes - read
if remainder > int64(maxInt) {
read += int64(maxInt)
cb(t.Name, t.Object.Size, read, maxInt)
} else {
read += remainder
cb(t.Name, t.Object.Size, read, int(remainder))
}
}
}
}

16
transfer/basic_download.go
View File

@ -146,21 +146,7 @@ func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback
}
if rangeRequestOk {
tracerx.Printf("xfer: server accepted resume download request: %q from byte %d", t.Object.Oid, fromByte)
// Advance progress callback; must split into max int sizes though
if cb != nil {
const maxInt = int(^uint(0) >> 1)
for read := int64(0); read < fromByte; {
remainder := fromByte - read
if remainder > int64(maxInt) {
read += int64(maxInt)
cb(t.Name, t.Object.Size, read, maxInt)
} else {
read += remainder
cb(t.Name, t.Object.Size, read, int(remainder))
}
}
}
advanceCallbackProgress(cb, t, fromByte)
} else {
// Abort resume, perform regular download
tracerx.Printf("xfer: failed to resume download for %q from byte %d: %s. Re-downloading from start", t.Object.Oid, fromByte, failReason)

166
transfer/tus_upload.go Normal file
View File

@ -0,0 +1,166 @@
package transfer
import (
"fmt"
"io"
"io/ioutil"
"os"
"strconv"
"github.com/github/git-lfs/api"
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/httputil"
"github.com/github/git-lfs/progress"
"github.com/rubyist/tracerx"
)
const (
TusAdapterName = "tus"
TusVersion = "1.0.0"
)
// Adapter for tus.io protocol resumaable uploads
type tusUploadAdapter struct {
*adapterBase
}
func (a *tusUploadAdapter) ClearTempStorage() error {
// nothing to do, all temp state is on the server end
return nil
}
func (a *tusUploadAdapter) DoTransfer(t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
rel, ok := t.Object.Rel("upload")
if !ok {
return fmt.Errorf("No upload action for this object.")
}
// Note not supporting the Creation extension since the batch API generates URLs
// Also not supporting Concatenation to support parallel uploads of chunks; forward only
// 1. Send HEAD request to determine upload start point
// Request must include Tus-Resumable header (version)
tracerx.Printf("xfer: sending tus.io HEAD request for %q", t.Object.Oid)
req, err := httputil.NewHttpRequest("HEAD", rel.Href, rel.Header)
if err != nil {
return err
}
req.Header.Set("Tus-Resumable", TusVersion)
res, err := httputil.DoHttpRequest(req, false)
if err != nil {
return errutil.NewRetriableError(err)
}
// Response will contain Upload-Offset if supported
offHdr := res.Header.Get("Upload-Offset")
if len(offHdr) == 0 {
return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href)
}
offset, err := strconv.ParseInt(offHdr, 10, 64)
if err != nil || offset < 0 {
return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href)
}
// Upload-Offset=size means already completed (skip)
// Batch API will probably already detect this, but handle just in case
if offset >= t.Object.Size {
tracerx.Printf("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Object.Oid)
advanceCallbackProgress(cb, t, t.Object.Size)
return nil
}
// Open file for uploading
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
if err != nil {
return errutil.Error(err)
}
defer f.Close()
// Upload-Offset=0 means start from scratch, but still send PATCH
if offset == 0 {
tracerx.Printf("xfer: tus.io uploading %q from start", t.Object.Oid)
} else {
tracerx.Printf("xfer: tus.io resuming upload %q from %d", t.Object.Oid, offset)
advanceCallbackProgress(cb, t, offset)
_, err := f.Seek(offset, os.SEEK_CUR)
if err != nil {
return errutil.Error(err)
}
}
// 2. Send PATCH request with byte start point (even if 0) in Upload-Offset
// Response status must be 204
// Response Upload-Offset must be request Upload-Offset plus sent bytes
// Response may include Upload-Expires header in which case check not passed
tracerx.Printf("xfer: sending tus.io PATCH request for %q", t.Object.Oid)
req, err = httputil.NewHttpRequest("PATCH", rel.Href, rel.Header)
if err != nil {
return err
}
req.Header.Set("Tus-Resumable", TusVersion)
req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10))
req.Header.Set("Content-Type", "application/offset+octet-stream")
req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size-offset, 10))
req.ContentLength = t.Object.Size - offset
// Ensure progress callbacks made while uploading
// Wrap callback to give name context
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
if cb != nil {
return cb(t.Name, totalSize, readSoFar, readSinceLast)
}
return nil
}
var reader io.Reader
reader = &progress.CallbackReader{
C: ccb,
TotalSize: t.Object.Size,
Reader: f,
}
// Signal auth was ok on first read; this frees up other workers to start
if authOkFunc != nil {
reader = newStartCallbackReader(reader, func(*startCallbackReader) {
authOkFunc()
})
}
req.Body = ioutil.NopCloser(reader)
res, err = httputil.DoHttpRequest(req, false)
if err != nil {
return errutil.NewRetriableError(err)
}
httputil.LogTransfer("lfs.data.upload", res)
// A status code of 403 likely means that an authentication token for the
// upload has expired. This can be safely retried.
if res.StatusCode == 403 {
return errutil.NewRetriableError(err)
}
if res.StatusCode > 299 {
return errutil.Errorf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
}
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
return api.VerifyUpload(t.Object)
}
func init() {
newfunc := func(name string, dir Direction) TransferAdapter {
switch dir {
case Upload:
bu := &tusUploadAdapter{newAdapterBase(name, dir, nil)}
// self implements impl
bu.transferImpl = bu
return bu
case Download:
panic("Should never ask tus.io to download")
}
return nil
}
RegisterNewTransferAdapterFunc(TusAdapterName, Upload, newfunc)
}