Merge branch 'master' into multitransfer

Conflicts:
	lfs/pointer_smudge.go
	lfs/upload_queue.go

.gitignore

@@ -8,3 +8,5 @@ servertest
 man/*
 
 *.test
+tmp
+test/remote

commands/command_clean.go

@@ -1,9 +1,10 @@
 package commands
 
 import (
+	"os"
+
 	"github.com/github/git-lfs/lfs"
 	"github.com/spf13/cobra"
-	"os"
 )
 
 var (

commands/command_get.go

@@ -1,10 +1,11 @@
 package commands
 
 import (
+	"os"
+
 	"github.com/github/git-lfs/git"
 	"github.com/github/git-lfs/lfs"
 	"github.com/spf13/cobra"
-	"os"
 )
 
 var (

commands/command_logs.go

@@ -2,11 +2,12 @@ package commands
 
 import (
 	"errors"
-	"github.com/github/git-lfs/lfs"
-	"github.com/spf13/cobra"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+
+	"github.com/github/git-lfs/lfs"
+	"github.com/spf13/cobra"
 )
 
 var (

commands/command_pointer.go

@@ -6,11 +6,12 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"github.com/github/git-lfs/lfs"
-	"github.com/spf13/cobra"
 	"io"
 	"os"
 	"os/exec"
+
+	"github.com/github/git-lfs/lfs"
+	"github.com/spf13/cobra"
 )
 
 var (

commands/command_pre_push.go

@@ -1,11 +1,12 @@
 package commands
 
 import (
-	"github.com/github/git-lfs/lfs"
-	"github.com/spf13/cobra"
 	"io/ioutil"
 	"os"
 	"strings"
+
+	"github.com/github/git-lfs/lfs"
+	"github.com/spf13/cobra"
 )
 
 var (

commands/command_push.go

@@ -1,13 +1,14 @@
 package commands
 
 import (
+	"io/ioutil"
+	"os"
+	"strings"
+
 	"github.com/github/git-lfs/git"
 	"github.com/github/git-lfs/lfs"
 	"github.com/rubyist/tracerx"
 	"github.com/spf13/cobra"
-	"io/ioutil"
-	"os"
-	"strings"
 )
 
 var (

commands/command_smudge.go

@@ -2,11 +2,12 @@ package commands
 
 import (
 	"bytes"
-	"github.com/github/git-lfs/lfs"
-	"github.com/spf13/cobra"
 	"io"
 	"os"
 	"path/filepath"
+
+	"github.com/github/git-lfs/lfs"
+	"github.com/spf13/cobra"
 )
 
 var (

commands/command_status.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"fmt"
+
 	"github.com/github/git-lfs/git"
 	"github.com/github/git-lfs/lfs"
 	"github.com/spf13/cobra"

commands/command_track.go

@@ -3,12 +3,13 @@ package commands
 import (
 	"bufio"
 	"fmt"
-	"github.com/github/git-lfs/lfs"
-	"github.com/spf13/cobra"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/github/git-lfs/lfs"
+	"github.com/spf13/cobra"
 )
 
 var (

commands/command_untrack.go

@@ -2,11 +2,12 @@ package commands
 
 import (
 	"bufio"
-	"github.com/github/git-lfs/lfs"
-	"github.com/spf13/cobra"
 	"io/ioutil"
 	"os"
 	"strings"
+
+	"github.com/github/git-lfs/lfs"
+	"github.com/spf13/cobra"
 )
 
 var (

commands/commands.go

@@ -3,8 +3,6 @@ package commands
 import (
 	"bytes"
 	"fmt"
-	"github.com/github/git-lfs/lfs"
-	"github.com/spf13/cobra"
 	"io"
 	"log"
 	"os"
@@ -12,6 +10,9 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/github/git-lfs/lfs"
+	"github.com/spf13/cobra"
 )
 
 var (
@@ -65,7 +66,7 @@ func LoggedError(err error, format string, args ...interface{}) {
 	file := handlePanic(err)
 
 	if len(file) > 0 {
-		fmt.Fprintf(os.Stderr, "\nErrors logged to %s.\nUse `git lfs logs last` to view the log.\n", file)
+		fmt.Fprintf(os.Stderr, "\nErrors logged to %s\nUse `git lfs logs last` to view the log.\n", file)
 	}
 }
 
@@ -105,7 +106,60 @@ func handlePanic(err error) string {
 		return ""
 	}
 
-	return logPanic(err, false)
+	return logPanic(err)
+}
+
+func logPanic(loggedError error) string {
+	var fmtWriter io.Writer = os.Stderr
+
+	now := time.Now()
+	name := now.Format("20060102T150405.999999999")
+	full := filepath.Join(lfs.LocalLogDir, name+".log")
+
+	if err := os.MkdirAll(lfs.LocalLogDir, 0755); err != nil {
+		full = ""
+		fmt.Fprintf(fmtWriter, "Unable to log panic to %s: %s\n\n", lfs.LocalLogDir, err.Error())
+	} else if file, err := os.Create(full); err != nil {
+		filename := full
+		full = ""
+		defer func() {
+			fmt.Fprintf(fmtWriter, "Unable to log panic to %s\n\n", filename)
+			logPanicToWriter(fmtWriter, err)
+		}()
+	} else {
+		fmtWriter = file
+		defer file.Close()
+	}
+
+	logPanicToWriter(fmtWriter, loggedError)
+
+	return full
+}
+
+func logPanicToWriter(w io.Writer, loggedError error) {
+	fmt.Fprintf(w, "> %s", filepath.Base(os.Args[0]))
+	if len(os.Args) > 0 {
+		fmt.Fprintf(w, " %s", strings.Join(os.Args[1:], " "))
+	}
+	fmt.Fprintln(w)
+
+	logEnv(w)
+	fmt.Fprintln(w)
+
+	w.Write(ErrorBuffer.Bytes())
+	fmt.Fprintln(w)
+
+	fmt.Fprintln(w, loggedError.Error())
+
+	if wErr, ok := loggedError.(ErrorWithStack); ok {
+		fmt.Fprintln(w, wErr.InnerError())
+		for key, value := range wErr.Context() {
+			fmt.Fprintf(w, "%s=%s\n", key, value)
+		}
+		w.Write(wErr.Stack())
+	} else {
+		w.Write(lfs.Stack())
+	}
 }
 
 func logEnv(w io.Writer) {
@@ -114,56 +168,6 @@ func logEnv(w io.Writer) {
 	}
 }
 
-func logPanic(loggedError error, recursive bool) string {
-	var fmtWriter io.Writer = os.Stderr
-
-	if err := os.MkdirAll(lfs.LocalLogDir, 0755); err != nil {
-		fmt.Fprintf(fmtWriter, "Unable to log panic to %s: %s\n\n", lfs.LocalLogDir, err.Error())
-		return ""
-	}
-
-	now := time.Now()
-	name := now.Format("20060102T150405.999999999")
-	full := filepath.Join(lfs.LocalLogDir, name+".log")
-
-	file, err := os.Create(full)
-	if err == nil {
-		fmtWriter = file
-		defer file.Close()
-	}
-
-	fmt.Fprintf(fmtWriter, "> %s", filepath.Base(os.Args[0]))
-	if len(os.Args) > 0 {
-		fmt.Fprintf(fmtWriter, " %s", strings.Join(os.Args[1:], " "))
-	}
-	fmt.Fprint(fmtWriter, "\n")
-
-	logEnv(fmtWriter)
-	fmt.Fprint(fmtWriter, "\n")
-
-	fmtWriter.Write(ErrorBuffer.Bytes())
-	fmt.Fprint(fmtWriter, "\n")
-
-	fmt.Fprintln(fmtWriter, loggedError.Error())
-
-	if wErr, ok := loggedError.(ErrorWithStack); ok {
-		fmt.Fprintln(fmtWriter, wErr.InnerError())
-		for key, value := range wErr.Context() {
-			fmt.Fprintf(fmtWriter, "%s=%s\n", key, value)
-		}
-		fmtWriter.Write(wErr.Stack())
-	} else {
-		fmtWriter.Write(lfs.Stack())
-	}
-
-	if err != nil && !recursive {
-		fmt.Fprintf(fmtWriter, "Unable to log panic to %s\n\n", full)
-		logPanic(err, true)
-	}
-
-	return full
-}
-
 type ErrorWithStack interface {
 	Context() map[string]string
 	InnerError() string

commands/commands_test.go

@@ -3,7 +3,6 @@ package commands
 import (
 	"errors"
 	"fmt"
-	"github.com/bmizerany/assert"
 	"io"
 	"io/ioutil"
 	"os"
@@ -11,6 +10,8 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/bmizerany/assert"
 )
 
 var (

commands/init_test.go

@@ -1,12 +1,13 @@
 package commands
 
 import (
-	"github.com/bmizerany/assert"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/bmizerany/assert"
 )
 
 func TestInit(t *testing.T) {

commands/pointer_test.go

@@ -1,12 +1,13 @@
 package commands
 
 import (
-	"github.com/bmizerany/assert"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/bmizerany/assert"
 )
 
 func TestPointerWithBuildAndCompareStdinMismatch(t *testing.T) {

commands/smudge_test.go

@@ -2,11 +2,12 @@ package commands
 
 import (
 	"bytes"
-	"github.com/bmizerany/assert"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/bmizerany/assert"
 )
 
 func TestSmudge(t *testing.T) {

commands/track_test.go

@@ -1,11 +1,12 @@
 package commands
 
 import (
-	"github.com/bmizerany/assert"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/bmizerany/assert"
 )
 
 func TestTrack(t *testing.T) {

commands/version_test.go

@@ -2,8 +2,9 @@ package commands
 
 import (
 	"fmt"
-	"github.com/github/git-lfs/lfs"
 	"testing"
+
+	"github.com/github/git-lfs/lfs"
 )
 
 func TestVersionOnEmptyRepository(t *testing.T) {

docs/linux-build.md

@@ -1,43 +1,12 @@
 ## Building on Linux
 
-## Ubuntu 14.04 (Trusty Tahr)
+There are build scripts for recent versions of CentOS- and Debian-flavored
+Linuxes in `../scripts/{centos,debian}-build`. Both install all prerequisites,
+then build the client and the man pages in Docker containers for CentOS 7,
+Debian 8, and Ubuntu 14.04.
 
-### Building
+On CentOS 6, the client builds, but not the man pages, because of problems
+getting the right version of Ruby.
 
-```
+Earlier versions of CentOS and Debian/Ubuntu have trouble building go, so they
-sudo apt-get install golang-go git
+are non-starters.
-
-./script/bootstrap
-```
-
-That will place a git-lfs binary in the `bin/` directory. Copy the binary to a directory in your path:
-
-```
-sudo cp bin/git-lfs /usr/local/bin
-```
-
-Try it:
-
-```
-[949][rubiojr@octox] git lfs
-git-lfs v0.0.1
-
-[~]
-[949][rubiojr@octox] git lfs init
-git lfs initialized
-```
-
-### Installing the man pages
-
-You'll need ruby and rubygems to install the `ronn` gem:
-
-
-```
-sudo apt-get install ruby build-essential
-sudo gem install ronn
-./script/man
-sudo mkdir -p /usr/local/share/man/man1
-sudo cp man/*.1 /usr/local/share/man/man1
-```
-
-`git help lfs` should show the git-lfs man pages now.

docs/spec.md

@@ -171,7 +171,7 @@ exist, and the sha-256 signature of the contents matches the given OID.
 * Write the pointer file to STDOUT.
 
 Note that the `clean` filter does not push the file to the server.  Use the
-`git lfs sync` command to do that.
+`git push` command to do that (lfs files are pushed before commits in a pre-push hook).
 
 The `smudge` filter runs as files are being checked out from the Git repository
 to the working directory.  Git sends the content of the Git blob as STDIN, and

git/git.go

@@ -4,10 +4,11 @@ package git
 import (
 	"errors"
 	"fmt"
-	"github.com/rubyist/tracerx"
 	"io"
 	"os/exec"
 	"strings"
+
+	"github.com/rubyist/tracerx"
 )
 
 func LsRemote(remote, remoteRef string) (string, error) {
@@ -108,7 +109,8 @@ func simpleExec(stdin io.Reader, name string, arg ...string) (string, error) {
 	output, err := cmd.Output()
 	if _, ok := err.(*exec.ExitError); ok {
 		return "", nil
-	} else if err != nil {
+	}
+	if err != nil {
 		return fmt.Sprintf("Error running %s %s", name, arg), err
 	}
 

lfs/client.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/rubyist/tracerx"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -14,6 +13,8 @@ import (
 	"path/filepath"
 	"regexp"
 	"strconv"
+
+	"github.com/rubyist/tracerx"
 )
 
 const (
@@ -447,42 +448,42 @@ func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedErro
 func doApiRequestWithRedirects(req *http.Request, creds Creds, via []*http.Request) (*http.Response, *WrappedError) {
 	res, wErr := doHttpRequest(req, creds)
 	if wErr != nil {
-		return res, wErr
+		return nil, wErr
 	}
 
 	if res.StatusCode == 307 {
 		redirectedReq, redirectedCreds, err := newClientRequest(req.Method, res.Header.Get("Location"))
 		if err != nil {
-			return res, Errorf(err, err.Error())
+			return nil, Errorf(err, err.Error())
 		}
 
 		via = append(via, req)
-		if seeker, ok := req.Body.(io.Seeker); ok {
+		seeker, ok := req.Body.(io.Seeker)
-			_, err := seeker.Seek(0, 0)
+		if !ok {
-			if err != nil {
+			return nil, Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.")
-				return res, Error(err)
-			}
-			redirectedReq.Body = req.Body
-			redirectedReq.ContentLength = req.ContentLength
-		} else {
-			return res, Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.")
 		}
 
+		if _, err := seeker.Seek(0, 0); err != nil {
+			return nil, Error(err)
+		}
+		redirectedReq.Body = req.Body
+		redirectedReq.ContentLength = req.ContentLength
+
 		if err = checkRedirect(redirectedReq, via); err != nil {
-			return res, Errorf(err, err.Error())
+			return nil, Errorf(err, err.Error())
 		}
 
 		return doApiRequestWithRedirects(redirectedReq, redirectedCreds, via)
 	}
 
-	return res, wErr
+	return res, nil
 }
 
 func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) {
 	via := make([]*http.Request, 0, 4)
 	res, wErr := doApiRequestWithRedirects(req, creds, via)
 	if wErr != nil {
-		return res, nil, wErr
+		return nil, nil, wErr
 	}
 
 	obj := &objectResource{}
@@ -490,9 +491,10 @@ func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResour
 
 	if wErr != nil {
 		setErrorResponseContext(wErr, res)
+		return nil, nil, wErr
 	}
 
-	return res, obj, wErr
+	return res, obj, nil
 }
 
 func doApiBatchRequest(req *http.Request, creds Creds) (*http.Response, []*objectResource, *WrappedError) {
@@ -608,26 +610,33 @@ func newApiRequest(method, oid string) (*http.Request, Creds, error) {
 	}
 
 	req, creds, err := newClientRequest(method, u.String())
-	if err == nil {
+	if err != nil {
-		req.Header.Set("Accept", mediaType)
+		return nil, nil, err
-		if res.Header != nil {
+	}
-			for key, value := range res.Header {
+
-				req.Header.Set(key, value)
+	req.Header.Set("Accept", mediaType)
-			}
+	if res.Header != nil {
+		for key, value := range res.Header {
+			req.Header.Set(key, value)
 		}
 	}
-	return req, creds, err
+
+	return req, creds, nil
 }
 
 func newClientRequest(method, rawurl string) (*http.Request, Creds, error) {
 	req, err := http.NewRequest(method, rawurl, nil)
 	if err != nil {
-		return req, nil, err
+		return nil, nil, err
 	}
 
 	req.Header.Set("User-Agent", UserAgent)
 	creds, err := getCreds(req)
-	return req, creds, err
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return req, creds, nil
 }
 
 func getCreds(req *http.Request) (Creds, error) {
@@ -656,18 +665,18 @@ func getCreds(req *http.Request) (Creds, error) {
 	return nil, nil
 }
 
-func setErrorRequestContext(err *WrappedError, req *http.Request) {
-	err.Set("Endpoint", Config.Endpoint().Url)
-	err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
-	setErrorHeaderContext(err, "Response", req.Header)
-}
-
 func setErrorResponseContext(err *WrappedError, res *http.Response) {
 	err.Set("Status", res.Status)
 	setErrorHeaderContext(err, "Request", res.Header)
 	setErrorRequestContext(err, res.Request)
 }
 
+func setErrorRequestContext(err *WrappedError, req *http.Request) {
+	err.Set("Endpoint", Config.Endpoint().Url)
+	err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
+	setErrorHeaderContext(err, "Response", req.Header)
+}
+
 func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) {
 	for key, _ := range head {
 		contextKey := fmt.Sprintf("%s:%s", prefix, key)

lfs/config.go

@@ -2,7 +2,6 @@ package lfs
 
 import (
 	"fmt"
-	"github.com/github/git-lfs/git"
 	"net/http"
 	"net/url"
 	"os"
@@ -11,6 +10,8 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+
+	"github.com/github/git-lfs/git"
 )
 
 type Configuration struct {

lfs/config_test.go

@@ -1,8 +1,9 @@
 package lfs
 
 import (
-	"github.com/bmizerany/assert"
 	"testing"
+
+	"github.com/bmizerany/assert"
 )
 
 func TestEndpointDefaultsToOrigin(t *testing.T) {

lfs/http.go

@@ -4,13 +4,14 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
-	"github.com/rubyist/tracerx"
 	"io"
 	"net"
 	"net/http"
 	"os"
 	"strings"
 	"time"
+
+	"github.com/rubyist/tracerx"
 )
 
 func DoHTTP(c *Configuration, req *http.Request) (*http.Response, error) {
@@ -24,24 +25,29 @@ func DoHTTP(c *Configuration, req *http.Request) (*http.Response, error) {
 }
 
 func (c *Configuration) HttpClient() *http.Client {
-	if c.httpClient == nil {
+	if c.httpClient != nil {
-		tr := &http.Transport{
+		return c.httpClient
-			Proxy: http.ProxyFromEnvironment,
-			Dial: (&net.Dialer{
-				Timeout:   5 * time.Second,
-				KeepAlive: 30 * time.Second,
-			}).Dial,
-			TLSHandshakeTimeout: 5 * time.Second,
-		}
-		sslVerify, _ := c.GitConfig("http.sslverify")
-		if sslVerify == "false" || len(os.Getenv("GIT_SSL_NO_VERIFY")) > 0 {
-			tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-		}
-		c.httpClient = &http.Client{
-			Transport:     tr,
-			CheckRedirect: checkRedirect,
-		}
 	}
+
+	tr := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		Dial: (&net.Dialer{
+			Timeout:   5 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).Dial,
+		TLSHandshakeTimeout: 5 * time.Second,
+	}
+
+	sslVerify, _ := c.GitConfig("http.sslverify")
+	if sslVerify == "false" || len(os.Getenv("GIT_SSL_NO_VERIFY")) > 0 {
+		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	c.httpClient = &http.Client{
+		Transport:     tr,
+		CheckRedirect: checkRedirect,
+	}
+
 	return c.httpClient
 }
 

lfs/lfs.go

@@ -2,13 +2,14 @@ package lfs
 
 import (
 	"fmt"
-	"github.com/github/git-lfs/git"
-	"github.com/rubyist/tracerx"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
+
+	"github.com/github/git-lfs/git"
+	"github.com/rubyist/tracerx"
 )
 
 const Version = "0.5.1"
@@ -129,15 +130,16 @@ func recursiveResolveGitDir(dir string) (string, string, error) {
 	}
 
 	gitDir := filepath.Join(dir, gitExt)
-	if info, err := os.Stat(gitDir); err == nil {
+	info, err := os.Stat(gitDir)
-		if info.IsDir() {
+	if err != nil {
-			return dir, gitDir, nil
+		return recursiveResolveGitDir(filepath.Dir(dir))
-		} else {
-			return processDotGitFile(gitDir)
-		}
 	}
 
-	return recursiveResolveGitDir(filepath.Dir(dir))
+	if info.IsDir() {
+		return dir, gitDir, nil
+	}
+
+	return processDotGitFile(gitDir)
 }
 
 func processDotGitFile(file string) (string, string, error) {

lfs/pointer.go

@@ -115,11 +115,11 @@ func decodeKV(data []byte) (*Pointer, error) {
 	sizeStr, ok := parsed["size"]
 	if !ok {
 		return nil, errors.New("Invalid Oid")
-	} else {
+	}
-		size, err = strconv.ParseInt(sizeStr, 10, 0)
+
-		if err != nil {
+	size, err = strconv.ParseInt(sizeStr, 10, 0)
-			return nil, errors.New("Invalid size: " + sizeStr)
+	if err != nil {
-		}
+		return nil, errors.New("Invalid size: " + sizeStr)
 	}
 
 	return NewPointer(oid, size), nil

lfs/pointer_smudge.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/cheggaaa/pb"
 	"github.com/rubyist/tracerx"
-	"github.com/technoweenie/go-contentaddressable"
+	contentaddressable "github.com/technoweenie/go-contentaddressable"
 )
 
 func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, cb CopyCallback) error {

lfs/pointer_test.go

@@ -3,9 +3,10 @@ package lfs
 import (
 	"bufio"
 	"bytes"
-	"github.com/bmizerany/assert"
 	"strings"
 	"testing"
+
+	"github.com/bmizerany/assert"
 )
 
 func TestEncode(t *testing.T) {

lfs/scanner.go

@@ -3,13 +3,14 @@ package lfs
 import (
 	"bufio"
 	"bytes"
-	"github.com/rubyist/tracerx"
 	"io"
 	"os/exec"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/rubyist/tracerx"
 )
 
 const (
@@ -52,7 +53,11 @@ var z40 = regexp.MustCompile(`\^?0{40}`)
 // for all Git LFS pointers it finds for that ref.
 func ScanRefs(refLeft, refRight string) ([]*wrappedPointer, error) {
 	nameMap := make(map[string]string, 0)
+
 	start := time.Now()
+	defer func() {
+		tracerx.PerformanceSince("scan", start)
+	}()
 
 	revs, err := revListShas(refLeft, refRight, refLeft == "", nameMap)
 	if err != nil {
@@ -77,8 +82,6 @@ func ScanRefs(refLeft, refRight string) ([]*wrappedPointer, error) {
 		pointers = append(pointers, p)
 	}
 
-	tracerx.PerformanceSince("scan", start)
-
 	return pointers, nil
 }
 
@@ -86,7 +89,11 @@ func ScanRefs(refLeft, refRight string) ([]*wrappedPointer, error) {
 // Git LFS pointers it finds in the index.
 func ScanIndex() ([]*wrappedPointer, error) {
 	nameMap := make(map[string]*indexFile, 0)
+
 	start := time.Now()
+	defer func() {
+		tracerx.PerformanceSince("scan-staging", start)
+	}()
 
 	revs, err := revListIndex(false, nameMap)
 	if err != nil {
@@ -135,8 +142,6 @@ func ScanIndex() ([]*wrappedPointer, error) {
 		pointers = append(pointers, p)
 	}
 
-	tracerx.PerformanceSince("scan-staging", start)
-
 	return pointers, nil
 
 }

lfs/setup.go

@@ -3,13 +3,14 @@ package lfs
 import (
 	"errors"
 	"fmt"
-	"github.com/github/git-lfs/git"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
+
+	"github.com/github/git-lfs/git"
 )
 
 var (
@@ -45,11 +46,9 @@ func InstallHooks(force bool) error {
 	hookPath := filepath.Join(LocalGitDir, "hooks", "pre-push")
 	if _, err := os.Stat(hookPath); err == nil && !force {
 		return upgradeHookOrError(hookPath, "pre-push", prePushHook, prePushUpgrades)
-	} else {
-		return ioutil.WriteFile(hookPath, []byte(prePushHook+"\n"), 0755)
 	}
 
-	return nil
+	return ioutil.WriteFile(hookPath, []byte(prePushHook+"\n"), 0755)
 }
 
 func upgradeHookOrError(hookPath, hookName, hook string, upgrades map[string]bool) error {
@@ -77,15 +76,16 @@ func upgradeHookOrError(hookPath, hookName, hook string, upgrades map[string]boo
 }
 
 func InstallFilters() error {
-	var err error
+	if err := setFilter("clean"); err != nil {
-	err = setFilter("clean")
+		return err
-	if err == nil {
-		err = setFilter("smudge")
 	}
-	if err == nil {
+	if err := setFilter("smudge"); err != nil {
-		err = requireFilters()
+		return err
 	}
-	return err
+	if err := requireFilters(); err != nil {
+		return err
+	}
+	return nil
 }
 
 func setFilter(filterName string) error {

lfs/ssh.go

@@ -2,8 +2,9 @@ package lfs
 
 import (
 	"encoding/json"
-	"github.com/rubyist/tracerx"
 	"os/exec"
+
+	"github.com/rubyist/tracerx"
 )
 
 type sshAuthResponse struct {

lfs/transfer_queue.go

@@ -2,9 +2,10 @@ package lfs
 
 import (
 	"fmt"
-	"github.com/cheggaaa/pb"
 	"sync"
 	"sync/atomic"
+
+	"github.com/cheggaaa/pb"
 )
 
 type Transferable interface {

lfs/util.go

@@ -80,9 +80,9 @@ func CopyCallbackFile(event, filename string, index, totalFiles int) (CopyCallba
 }
 
 func wrapProgressError(err error, event, filename string) error {
-	if err == nil {
+	if err != nil {
-		return nil
+		return fmt.Errorf("Error writing Git LFS %s progress to %s: %s", event, filename, err.Error())
 	}
 
-	return fmt.Errorf("Error writing Git LFS %s progress to %s: %s", event, filename, err.Error())
+	return nil
 }

lfs/util_test.go

@@ -2,9 +2,10 @@ package lfs
 
 import (
 	"bytes"
-	"github.com/bmizerany/assert"
 	"io/ioutil"
 	"testing"
+
+	"github.com/bmizerany/assert"
 )
 
 func TestWriterWithCallback(t *testing.T) {

script/build.go

@@ -4,11 +4,13 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/github/git-lfs/lfs"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
+
+	"github.com/github/git-lfs/lfs"
 )
 
 var (
@@ -61,27 +63,23 @@ func mainBuild() {
 	if !errored {
 		by, err := json.Marshal(buildMatrix)
 		if err != nil {
-			fmt.Println("Error encoding build matrix to json:", err)
+			log.Fatalln("Error encoding build matrix to json:", err)
-			os.Exit(1)
 		}
 
 		file, err := os.Create("bin/releases/build_matrix.json")
 		if err != nil {
-			fmt.Println("Error creating build_matrix.json:", err)
+			log.Fatalln("Error creating build_matrix.json:", err)
-			os.Exit(1)
 		}
 
 		written, err := file.Write(by)
 		file.Close()
 
 		if err != nil {
-			fmt.Println("Error writing build_matrix.json", err)
+			log.Fatalln("Error writing build_matrix.json", err)
-			os.Exit(1)
 		}
 
 		if jsonSize := len(by); written != jsonSize {
-			fmt.Printf("Expected to write %d bytes, actually wrote %d.\n", jsonSize, written)
+			log.Fatalf("Expected to write %d bytes, actually wrote %d.\n", jsonSize, written)
-			os.Exit(1)
 		}
 	}
 }
@@ -109,13 +107,13 @@ func build(buildos, buildarch string, buildMatrix map[string]Release) error {
 	if addenv {
 		err := os.MkdirAll(dir, 0755)
 		if err != nil {
-			fmt.Println("Error setting up installer:\n", err.Error())
+			log.Println("Error setting up installer:\n", err.Error())
 			return err
 		}
 
 		err = setupInstaller(buildos, buildarch, dir, buildMatrix)
 		if err != nil {
-			fmt.Println("Error setting up installer:\n", err.Error())
+			log.Println("Error setting up installer:\n", err.Error())
 			return err
 		}
 	}

script/centos-build

@@ -0,0 +1,29 @@
+#!/bin/bash -eu
+#
+# This script works with CentOS 6 or 7
+# The CentOS 5 kernel is too old for go's liking.
+
+trap 'echo FAIL' ERR
+
+if grep -q ' 6' /etc/redhat-release; then
+  rpm -q epel-release || rpm -Uvh http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+fi
+
+yum install -y bison git golang make man
+cd /tmp
+[ -d git-lfs ] || git clone https://github.com/github/git-lfs
+cd git-lfs
+./script/bootstrap
+install -D bin/git-lfs /usr/local/bin
+git lfs init
+
+# I don't know how to install ruby2.0 on CentOS6 yet
+if grep -q ' 7' /etc/redhat-release; then
+  yum install ruby ruby-devel
+  gem install ronn
+  ./script/man
+  install -D man/*.1 /usr/local/share/man/man1
+  git help lfs > /dev/null
+fi
+
+echo SUCCESS

script/cibuild

@@ -1,6 +1,8 @@
 #!/bin/sh
+set -e
 
 git config user.name || git config --global user.name "Git LFS Tests"
 git config user.email || git config --global user.email "git-lfs@example.com"
 
 script/test
+script/integration

script/debian-build

@@ -0,0 +1,49 @@
+#!/bin/bash -eux
+#
+# This script works with Debian 8 and Ubuntu 14.04 (LTS)
+# Go seems to build poorly on Ubuntu 12.04 (LTS)
+
+trap 'echo FAIL' ERR
+
+apt-get -y update
+apt-get -y install binutils bison curl gcc git golang-go make
+go_version=$(go version | sed 's/go version go//; s/ .*//')
+if [[ $go_version < 1.3.1 ]]; then
+  [[ -r ~/.gvm/scripts/gvm ]] ||
+    bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/master/binscripts/gvm-installer)
+  set +u
+  source ~/.gvm/scripts/gvm
+  gvm install go1.4.2
+  gvm use go1.4.2
+  set -u
+fi
+
+cd /tmp
+[ -d git-lfs ] || git clone https://github.com/github/git-lfs
+cd git-lfs
+./script/bootstrap
+install -D bin/git-lfs /usr/local/bin
+git lfs init
+
+apt-get -y install build-essential groff man
+if grep -q Debian /etc/issue; then
+  apt-get install -y ruby ruby-dev
+elif grep -q Ubuntu /etc/issue; then
+  apt-get -y install ruby2.0 ruby2.0-dev
+  rm /usr/bin/ruby && sudo ln -s /usr/bin/ruby2.0 /usr/bin/ruby
+  rm -fr /usr/bin/gem && sudo ln -s /usr/bin/gem2.0 /usr/bin/gem
+else
+  {
+    echo "unknown Debian release"
+    cat /etc/issue
+  } >&2
+  exit 1
+fi
+
+gem install ronn
+./script/man
+install -d /usr/local/share/man/man1
+install man/*.1 /usr/local/share/man/man1
+git help lfs > /dev/null
+
+echo SUCCESS

script/fmt

@@ -1,10 +1,15 @@
 #!/bin/bash
 
+formatter=gofmt
+hash goimports 2>/dev/null && {
+  formatter=goimports
+}
+
 # don't run gofmt in these directories
 ignored=(/bin/ /docs/ /log/ /man/ /tmp/ .vendor)
 for i in */ ; do
   if [[ ! ${ignored[*]} =~ "/$i" ]]; then
-    gofmt -w -l "$@" "${i%?}"
+    $formatter -w -l "$@" "${i%?}"
   fi
 done
 

script/install.sh.example

@@ -1,16 +1,17 @@
+#!/bin/sh -eu
+
 prefix="/usr/local"
 
-if [ "$PREFIX" != "" ] ; then
+if [ "${PREFIX:-}" != "" ] ; then
-  prefix=$PREFIX
+  prefix=${PREFIX:-}
-elif [ "$BOXEN_HOME" != "" ] ; then
+elif [ "${BOXEN_HOME:-}" != "" ] ; then
-  prefix=$BOXEN_HOME
+  prefix=${BOXEN_HOME:-}
 fi
 
-mkdir -p $prefix/bin
-
 rm -rf $prefix/bin/git-lfs*
 for g in git*; do
-  cp $g "$prefix/bin/$g"
+  install -D $g "$prefix/bin/$g"
 done
 
+PATH+=:$prefix/bin
 git lfs init

script/integration

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+. "test/testenv.sh"
+set -e
+
+SHUTDOWN_LFS=no
+SHOW_LOGS=yes
+
+atexit() {
+  res=${1:-$?}
+  SHUTDOWN_LFS=yes
+  if [ "$res" = "0" ]; then
+    SHOW_LOGS=no
+  fi
+
+  if [ "$SHOW_LOGS" = "yes" ]; then
+    if [ -s "$REMOTEDIR/gitserver.log" ]; then
+      echo ""
+      echo "gitserver.log:"
+      cat "$REMOTEDIR/gitserver.log"
+    fi
+
+    echo ""
+    echo "env:"
+    env
+  fi
+
+  shutdown
+  exit $res
+}
+
+trap "atexit" EXIT
+
+if [ -s "$LFS_URL_FILE" ]; then
+  SHOW_LOGS=no
+  echo "$LFS_URL_FILE still exists!"
+  echo "Confirm other tests are done, and run:"
+  echo "  $ curl $(cat "$LFS_URL_FILE")/shutdown"
+  exit 1
+fi
+
+setup
+
+for file in test/test-*.sh; do
+  echo "0$(cat .$(basename $file).time 2>/dev/null || true) $file"
+done | sort -rnk1 | awk '{ print $2 }' | xargs -I % -P 4 -n 1 /bin/sh -c % --batch

script/release.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"log"
 	"net/url"
 	"os"
 	"os/exec"
@@ -17,24 +18,21 @@ var (
 
 func mainRelease() {
 	if *ReleaseId < 1 {
-		fmt.Println("Need a valid github/git-lfs release id.")
+		log.Println("Need a valid github/git-lfs release id.")
-		fmt.Println("usage: script/release -id")
+		log.Fatalln("usage: script/release -id")
-		os.Exit(1)
 	}
 
 	file, err := os.Open("bin/releases/build_matrix.json")
 	if err != nil {
-		fmt.Println("Error opening build_matrix.json:", err)
+		log.Println("Error opening build_matrix.json:", err)
-		fmt.Println("Ensure `script/bootstrap -all` has completed successfully")
+		log.Fatalln("Ensure `script/bootstrap -all` has completed successfully")
-		os.Exit(1)
 	}
 
 	defer file.Close()
 
 	buildMatrix := make(map[string]Release)
 	if err := json.NewDecoder(file).Decode(&buildMatrix); err != nil {
-		fmt.Println("Error reading build_matrix.json:", err)
+		log.Fatalln("Error reading build_matrix.json:", err)
-		os.Exit(1)
 	}
 
 	for _, rel := range buildMatrix {
@@ -62,8 +60,7 @@ func release(rel Release) {
 
 	by, err := cmd.Output()
 	if err != nil {
-		fmt.Println("Error running curl:", err)
+		log.Fatalln("Error running curl:", err)
-		os.Exit(1)
 	}
 
 	fmt.Println(string(by))

script/script.go

@@ -2,8 +2,7 @@ package main
 
 import (
 	"flag"
-	"fmt"
+	"log"
-	"os"
 )
 
 type Release struct {
@@ -21,7 +20,6 @@ func main() {
 	case "release":
 		mainRelease()
 	default:
-		fmt.Println("Unknown command:", *SubCommand)
+		log.Fatalln("Unknown command:", *SubCommand)
-		os.Exit(1)
 	}
 }

script/test

@@ -11,7 +11,7 @@ rm -f $LOCALSRCDIR
 ln -s `pwd` $LOCALSRCDIR
 
 GOPATH="`pwd`/.vendor"
-SUITE="./${1:-"..."}"
+SUITE="./${1:-"lfs ./commands"}"
 if [ $# -gt 0 ]; then
   shift
 fi

test/README.md

@@ -0,0 +1,123 @@
+# Git LFS Tests
+
+Git LFS uses two form of tests: unit tests for the internals written in Go, and
+integration tests that run `git` and `git-lfs` in a real shell environment.
+You can run them separately:
+
+```
+$ script/test        # Tests the Go packages.
+$ script/integration # Tests the commands in shell scripts.
+```
+
+CI servers should always run both:
+
+```
+$ script/cibuild
+```
+
+## Internal Package Tests
+
+The internal tests use Go's builtin [testing][t] package.
+
+You can run individual tests by passing arguments to `script/test`:
+
+```
+# test a specific Go package
+$ script/test lfs
+
+# pass other `go test` arguments
+$ script/test lfs -run TestSuccessStatus -v
+github.com/kr/text
+github.com/cheggaaa/pb
+github.com/rubyist/tracerx
+github.com/technoweenie/go-contentaddressable
+github.com/kr/pretty
+github.com/github/git-lfs/git
+github.com/bmizerany/assert
+=== RUN TestSuccessStatus
+--- PASS: TestSuccessStatus (0.00 seconds)
+PASS
+ok  	_/Users/rick/github/git-lfs/lfs	0.011s
+```
+
+[t]: http://golang.org/pkg/testing/
+
+## Integration Tests
+
+Git LFS integration tests are shell scripts that test the `git-lfs` command from
+the shell.  Each test file can be run individually, or in parallel through
+`script/integration`. Some tests will change the `pwd`, so it's important that
+they run in separate OS processes.
+
+```
+$ test/test-happy-path.sh
+compile git-lfs for test/test-happy-path.sh
+LFSTEST_URL=/Users/rick/github/git-lfs/test/remote/url LFSTEST_DIR=/Users/rick/github/git-lfs/test/remote lfstest-gitserver
+test: happy path ...                                               OK
+```
+
+1. The integration tests should not rely on global system or git config.
+2. The tests should be cross platform (Linux, Mac, Windows).
+3. Tests should bootstrap an isolated, clean environment.  See the Test Suite
+section.
+4. Successful test runs should have minimal output.
+5. Failing test runs should dump enough information to diagnose the bug.  This
+includes stdout, stderr, any log files, and even the OS environment.
+
+### Test Suite
+
+The `testenv.sh` script includes some global variables used in tests.  This
+should be automatically included in every `test/test-*.sh` script and
+`script/integration`.
+
+`testhelpers.sh` defines some shell functions.  Most are only used in the test
+scripts themselves.  `script/integration` uses the `setup()` and `shutdown()`
+functions.
+
+`testlib.sh` is a [fork of a lightweight shell testing lib][testlib] that is
+used internally at GitHub.  Only the `test/test-*.sh` scripts should include
+this.
+
+Tests live in this `./test` directory, and must have a unique name like:
+`test-{name}.sh`. All tests should start with a basic template.  See
+`test/test-happy-path.sh` for an example.
+
+```
+#!/bin/sh
+
+. "test/testlib.sh"
+
+begin_test "template"
+(
+  set -e
+
+  echo "your tests go here"
+)
+end_test
+```
+
+The `set -e` command will bail on the test at the first command that returns a
+non zero exit status. Use simple shell commands like `grep` as assertions.
+
+The test suite has standard `setup` and `shutdown` functions that should be
+run only once.  If a test script is run by `script/integration`, it will skip
+the functions.  Setup does the following:
+
+* Resets temporary test directories.
+* Compiles git-lfs with the latest code changes.
+* Compiles Go files in `test/cmd` to `bin`, and adds them the PATH.
+* Spins up a test Git and Git LFS server so the entire push/pull flow can be
+exercised.
+* Sets up a git credential helper that always returns a set username and
+password.
+
+The test Git server writes a `test/remote/url` file when it's complete.  This
+file is how individual test scripts detect if `script/integration` is being
+run.  You can fake this by manually spinning up the Git server using the
+`lfstest-gitserver` line that is output after Git LFS is compiled.
+
+By default, temporary directories in `tmp` and the `test/remote` directory are
+cleared after test runs. Send the "KEEPTRASH" if you want to keep these files
+around for debugging failed tests.
+
+[testlib]: https://gist3.github.com/rtomayko/3877539

test/cmd/git-credential-lfstest.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+)
+
+var (
+	commands = map[string]func(){
+		"get":   fill,
+		"store": noop,
+		"erase": noop,
+	}
+
+	delim = '\n'
+
+	hostRE = regexp.MustCompile(`\A127.0.0.1:\d+\z`)
+)
+
+func main() {
+	if argsize := len(os.Args); argsize != 2 {
+		fmt.Fprintf(os.Stderr, "wrong number of args: %d\n", argsize)
+		os.Exit(1)
+	}
+
+	arg := os.Args[1]
+	cmd := commands[arg]
+
+	if cmd == nil {
+		fmt.Fprintf(os.Stderr, "bad cmd: %s\n", arg)
+		os.Exit(1)
+	}
+
+	cmd()
+}
+
+func fill() {
+	scanner := bufio.NewScanner(os.Stdin)
+	creds := map[string]string{}
+	for scanner.Scan() {
+		line := scanner.Text()
+		parts := strings.SplitN(line, "=", 2)
+		if len(parts) != 2 {
+			fmt.Fprintf(os.Stderr, "bad line: %s\n", line)
+			os.Exit(1)
+		}
+
+		creds[parts[0]] = strings.TrimSpace(parts[1])
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintf(os.Stderr, "reading standard input: %v", err)
+		os.Exit(1)
+	}
+
+	if _, ok := creds["username"]; !ok {
+		creds["username"] = "user"
+	}
+
+	if _, ok := creds["password"]; !ok {
+		creds["password"] = "pass"
+	}
+
+	if host := creds["host"]; !hostRE.MatchString(host) {
+		fmt.Fprintf(os.Stderr, "invalid host: %s, should be '127.0.0.1:\\d+'\n", host)
+		os.Exit(1)
+	}
+
+	for key, value := range creds {
+		fmt.Fprintf(os.Stdout, "%s=%s\n", key, value)
+	}
+}
+
+func noop() {}

test/cmd/lfstest-gitserver.go

@@ -0,0 +1,238 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"net/textproto"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+var (
+	repoDir      string
+	largeObjects = make(map[string][]byte)
+	server       *httptest.Server
+)
+
+func main() {
+	repoDir = os.Getenv("LFSTEST_DIR")
+
+	mux := http.NewServeMux()
+	server = httptest.NewServer(mux)
+	stopch := make(chan bool)
+
+	mux.HandleFunc("/shutdown", func(w http.ResponseWriter, r *http.Request) {
+		stopch <- true
+	})
+
+	mux.HandleFunc("/storage/", storageHandler)
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if strings.Contains(r.URL.Path, "/info/lfs") {
+			log.Printf("git lfs %s %s\n", r.Method, r.URL)
+			lfsHandler(w, r)
+			return
+		}
+
+		log.Printf("git http-backend %s %s\n", r.Method, r.URL)
+		gitHandler(w, r)
+	})
+
+	urlname := os.Getenv("LFSTEST_URL")
+	if len(urlname) == 0 {
+		urlname = "lfstest-gitserver"
+	}
+
+	file, err := os.Create(urlname)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	file.Write([]byte(server.URL))
+	file.Close()
+	log.Println(server.URL)
+
+	defer func() {
+		os.RemoveAll(urlname)
+	}()
+
+	<-stopch
+	log.Println("git server done")
+}
+
+type lfsObject struct {
+	Oid   string             `json:"oid,omitempty"`
+	Size  int64              `json:"size,omitempty"`
+	Links map[string]lfsLink `json:"_links,omitempty"`
+}
+
+type lfsLink struct {
+	Href   string            `json:"href"`
+	Header map[string]string `json:"header,omitempty"`
+}
+
+// handles any requests with "{name}.server.git/info/lfs" in the path
+func lfsHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/vnd.git-lfs+json")
+	switch r.Method {
+	case "POST":
+		lfsPostHandler(w, r)
+	case "GET":
+		lfsGetHandler(w, r)
+	default:
+		w.WriteHeader(405)
+	}
+}
+
+func lfsPostHandler(w http.ResponseWriter, r *http.Request) {
+	buf := &bytes.Buffer{}
+	tee := io.TeeReader(r.Body, buf)
+	obj := &lfsObject{}
+	err := json.NewDecoder(tee).Decode(obj)
+	io.Copy(ioutil.Discard, r.Body)
+	r.Body.Close()
+
+	log.Println("REQUEST")
+	log.Println(buf.String())
+	log.Printf("OID: %s\n", obj.Oid)
+	log.Printf("Size: %d\n", obj.Size)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	res := &lfsObject{
+		Links: map[string]lfsLink{
+			"upload": lfsLink{
+				Href: server.URL + "/storage/" + obj.Oid,
+			},
+		},
+	}
+
+	by, err := json.Marshal(res)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Println("RESPONSE: 202")
+	log.Println(string(by))
+
+	w.WriteHeader(202)
+	w.Write(by)
+}
+
+func lfsGetHandler(w http.ResponseWriter, r *http.Request) {
+	parts := strings.Split(r.URL.Path, "/")
+	oid := parts[len(parts)-1]
+
+	by, ok := largeObjects[oid]
+	if !ok {
+		w.WriteHeader(404)
+		return
+	}
+
+	obj := &lfsObject{
+		Oid:  oid,
+		Size: int64(len(by)),
+		Links: map[string]lfsLink{
+			"download": lfsLink{
+				Href: server.URL + "/storage/" + oid,
+			},
+		},
+	}
+
+	by, err := json.Marshal(obj)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Println("RESPONSE: 200")
+	log.Println(string(by))
+
+	w.WriteHeader(200)
+	w.Write(by)
+}
+
+// handles any /storage/{oid} requests
+func storageHandler(w http.ResponseWriter, r *http.Request) {
+	log.Printf("storage %s %s\n", r.Method, r.URL)
+	switch r.Method {
+	case "PUT":
+		hash := sha256.New()
+		buf := &bytes.Buffer{}
+		io.Copy(io.MultiWriter(hash, buf), r.Body)
+		oid := hex.EncodeToString(hash.Sum(nil))
+		if !strings.HasSuffix(r.URL.Path, "/"+oid) {
+			w.WriteHeader(403)
+			return
+		}
+
+		largeObjects[oid] = buf.Bytes()
+
+	case "GET":
+		parts := strings.Split(r.URL.Path, "/")
+		oid := parts[len(parts)-1]
+
+		if by, ok := largeObjects[oid]; ok {
+			w.Write(by)
+			return
+		}
+
+		w.WriteHeader(404)
+	default:
+		w.WriteHeader(405)
+	}
+}
+
+func gitHandler(w http.ResponseWriter, r *http.Request) {
+	defer func() {
+		io.Copy(ioutil.Discard, r.Body)
+		r.Body.Close()
+	}()
+
+	cmd := exec.Command("git", "http-backend")
+	cmd.Env = []string{
+		fmt.Sprintf("GIT_PROJECT_ROOT=%s", repoDir),
+		fmt.Sprintf("GIT_HTTP_EXPORT_ALL="),
+		fmt.Sprintf("PATH_INFO=%s", r.URL.Path),
+		fmt.Sprintf("QUERY_STRING=%s", r.URL.RawQuery),
+		fmt.Sprintf("REQUEST_METHOD=%s", r.Method),
+		fmt.Sprintf("CONTENT_TYPE=%s", r.Header.Get("Content-Type")),
+	}
+
+	buffer := &bytes.Buffer{}
+	cmd.Stdin = r.Body
+	cmd.Stdout = buffer
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Run(); err != nil {
+		log.Fatal(err)
+	}
+
+	text := textproto.NewReader(bufio.NewReader(buffer))
+
+	code, _, _ := text.ReadCodeLine(-1)
+
+	if code != 0 {
+		w.WriteHeader(code)
+	}
+
+	headers, _ := text.ReadMIMEHeader()
+	head := w.Header()
+	for key, values := range headers {
+		for _, value := range values {
+			head.Add(key, value)
+		}
+	}
+
+	io.Copy(w, text.R)
+}

test/test-happy-path.sh

@@ -0,0 +1,71 @@
+#!/bin/sh
+# This is a sample Git LFS test.  See test/README.md and testhelpers.sh for
+# more documentation.
+
+. "test/testlib.sh"
+
+begin_test "happy path"
+(
+  set -e
+
+  # This initializes a new bare git repository in test/remote.
+  # These remote repositories are global to every test, so keep the names
+  # unique.
+  reponame="$(basename "$0" ".sh")"
+  setup_remote_repo "$reponame"
+
+  # Clone the repository from the test Git server.  This is empty, and will be
+  # used to test a "git pull" below. The repo is cloned to $TRASHDIR/clone
+  clone_repo "$reponame" clone
+
+  # Clone the repository again to $TRASHDIR/repo. This will be used to commit
+  # and push objects.
+  clone_repo "$reponame" repo
+
+  # This executes Git LFS from the local repo that was just cloned.
+  out=$($GITLFS track "*.dat" 2>&1)
+  echo "$out" | grep "Tracking \*.dat"
+
+  contents=$(printf "a")
+  contents_oid=$(printf "$contents" | shasum -a 256 | cut -f 1 -d " ")
+
+  # Regular Git commands can be used.
+  printf "$contents" > a.dat
+  git add a.dat
+  git add .gitattributes
+  out=$(git commit -m "add a.dat" 2>&1)
+  echo "$out" | grep "master (root-commit)"
+  echo "$out" | grep "2 files changed"
+  echo "$out" | grep "create mode 100644 a.dat"
+  echo "$out" | grep "create mode 100644 .gitattributes"
+
+  out=$(cat a.dat)
+  if [ "$out" != "a" ]; then
+    exit 1
+  fi
+
+  # This is a small shell function that runs several git commands together.
+  assert_pointer "master" "a.dat" "$contents_oid" 1
+
+  refute_server_object "$contents_oid"
+
+  # This pushes to the remote repository set up at the top of the test.
+  out=$(git push origin master 2>&1)
+  echo "$out" | grep "(1 of 1 files) 1 B / 1 B  100.00 %"
+  echo "$out" | grep "master -> master"
+
+  assert_server_object "$contents_oid" "$contents"
+
+  # change to the clone's working directory
+  cd ../clone
+
+  git pull 2>&1 | grep "Downloading a.dat (1 B)"
+
+  out=$(cat a.dat)
+  if [ "$out" != "a" ]; then
+    exit 1
+  fi
+
+  assert_pointer "master" "a.dat" "$contents_oid" 1
+)
+end_test

test/testenv.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+# Including in script/integration and every test/test-*.sh file.
+
+set -e
+
+# The root directory for the git-lfs repository
+ROOTDIR=$(cd $(dirname "$0")/.. && pwd)
+
+# Where Git LFS outputs the compiled binaries
+BINPATH="$ROOTDIR/bin"
+
+# Put bin path on PATH
+PATH="$BINPATH:$PATH"
+
+# create a temporary work space
+TMPDIR="$(cd $(dirname "$0")/.. && pwd)"/tmp
+
+# This is unique to every test file, and cleared after every test run.
+TRASHDIR="$TMPDIR/$(basename "$0")-$$"
+
+# Points to the git-lfs binary compiled just for the tests
+GITLFS="$BINPATH/git-lfs"
+
+# The directory that the test Git server works from.  This cleared at the
+# beginning of every test run.
+REMOTEDIR="$ROOTDIR/test/remote"
+
+# This is the prefix for Git config files.  See the "Test Suite" section in
+# test/README.md
+LFS_CONFIG="$REMOTEDIR/config"
+
+# This file contains the URL of the test Git server. See the "Test Suite"
+# section in test/README.md
+LFS_URL_FILE="$REMOTEDIR/url"
+
+mkdir -p "$TRASHDIR"
+
+. "test/testhelpers.sh"

test/testhelpers.sh

@@ -0,0 +1,144 @@
+#!/bin/sh
+
+# assert_pointer confirms that the pointer in the repository for $path in the
+# given $ref matches the given $oid and $size.
+#
+#   $ assert_pointer "master" "path/to/file" "some-oid" 123
+assert_pointer() {
+  local ref=$1
+  local path=$2
+  local oid=$3
+  local size=$4
+
+  gitblob=$(git ls-tree -l $ref | grep $path | cut -f 3 -d " ")
+  actual=$(git cat-file -p $gitblob)
+  expected=$(pointer $oid $size)
+
+  if [ "$expected" != "$actual" ]; then
+    exit 1
+  fi
+}
+
+# no-op.  check that the object does not exist in the git lfs server
+refute_server_object() {
+  echo "refute server object: no-op"
+}
+
+# no-op.  check that the object does exist in the git lfs server
+assert_server_object() {
+  echo "assert server object: no-op"
+}
+
+# pointer returns a string Git LFS pointer file.
+#
+#   $ pointer abc-some-oid 123
+#   > version ...
+pointer() {
+  local oid=$1
+  local size=$2
+  printf "version https://git-lfs.github.com/spec/v1
+oid sha256:%s
+size %s
+" "$oid" "$size"
+}
+
+# wait_for_file simply sleeps until a file exists.
+#
+#   $ wait_for_file "path/to/upcoming/file"
+wait_for_file() {
+  local filename=$1
+  n=0
+  while [ $n -lt 10 ]; do
+    if [ -s $filename ]; then
+      return 0
+    fi
+
+    sleep 0.5
+    n=`expr $n + 1`
+  done
+
+  return 1
+}
+
+# setup_remote_repo intializes a bare Git repository that is accessible through
+# the test Git server. The `pwd` is set to the repository's directory, in case
+# further commands need to be run. This server is running for every test in a
+# script/integration run, so every test file should setup its own remote
+# repository to avoid conflicts.
+#
+#   $ setup_remote_repo "some-name"
+#
+setup_remote_repo() {
+  local reponame=$1
+  echo "set up remote git repository: $reponame"
+  repodir="$REMOTEDIR/$reponame.git"
+  mkdir -p "$repodir"
+  cd "$repodir"
+  git init --bare
+  git config http.receivepack true
+  git config receive.denyCurrentBranch ignore
+
+  # dump a simple git config file so clones use this test's Git LFS command
+  # and the custom credential helper. This overrides any Git config that is
+  # already setup on the system.
+  printf "[filter \"lfs\"]
+	required = true
+	smudge = %s smudge %%f
+	clean = %s clean %%f
+[credential]
+	helper = %s
+[remote \"origin\"]
+	url = %s/%s
+	fetch = +refs/heads/*:refs/remotes/origin/*
+" "$GITLFS" "$GITLFS" lfstest "$GITSERVER" "$reponame" > "$LFS_CONFIG-$reponame"
+}
+
+# clone_repo clones a repository from the test Git server to the subdirectory
+# $dir under $TRASHDIR. setup_remote_repo() needs to be run first.
+clone_repo() {
+  cd "$TRASHDIR"
+
+  local reponame=$1
+  local dir=$2
+  echo "clone local git repository $reponame to $dir"
+  out=$(GIT_CONFIG="$LFS_CONFIG-$reponame" git clone "$GITSERVER/$reponame" "$dir" 2>&1)
+  cd "$dir"
+
+  git config credential.helper lfstest
+  echo "$out"
+}
+
+# setup initializes the clean, isolated environment for integration tests.
+setup() {
+  cd "$ROOTDIR"
+
+  rm -rf "test/remote"
+  mkdir "test/remote"
+
+  echo "compile git-lfs for $0"
+  script/bootstrap
+  $GITLFS version
+
+  for go in test/cmd/*.go; do
+    go build -o "$BINPATH/$(basename $go .go)" "$go"
+  done
+
+  echo "LFSTEST_URL=$LFS_URL_FILE LFSTEST_DIR=$REMOTEDIR lfstest-gitserver"
+  LFSTEST_URL="$LFS_URL_FILE" LFSTEST_DIR="$REMOTEDIR" lfstest-gitserver > "$REMOTEDIR/gitserver.log" 2>&1 &
+  wait_for_file "$LFS_URL_FILE"
+}
+
+# shutdown cleans the $TRASHDIR and shuts the test Git server down.
+shutdown() {
+  # every test/test-*.sh file should cleanup its trashdir
+  [ -z "$KEEPTRASH" ] && rm -rf "$TRASHDIR"
+
+  if [ "$SHUTDOWN_LFS" != "no" ]; then
+    # only cleanup test/remote after script/integration done OR a single
+    # test/test-*.sh file is run manually.
+    [ -z "$KEEPTRASH" ] && rm -rf "$REMOTEDIR"
+    if [ -s "$LFS_URL_FILE" ]; then
+      curl "$(cat "$LFS_URL_FILE")/shutdown"
+    fi
+  fi
+}

test/testlib.sh

@@ -0,0 +1,97 @@
+#!/bin/sh
+# Usage: . testlib.sh
+# Simple shell command language test library.
+#
+# Tests must follow the basic form:
+#
+#   begin_test "the thing"
+#   (
+#        set -e
+#        echo "hello"
+#        false
+#   )
+#   end_test
+#
+# When a test fails its stdout and stderr are shown.
+#
+# Note that tests must `set -e' within the subshell block or failed assertions
+# will not cause the test to fail and the result may be misreported.
+#
+# Copyright (c) 2011-13 by Ryan Tomayko <http://tomayko.com>
+# License: MIT
+
+. "test/testenv.sh"
+set -e
+
+# keep track of num tests and failures
+tests=0
+failures=0
+
+# this runs at process exit
+atexit () {
+  shutdown
+
+  if [ $failures -gt 0 ]; then
+    exit 1
+  else
+    exit 0
+  fi
+}
+
+# create the trash dir
+trap "atexit" EXIT
+
+SHUTDOWN_LFS=yes
+GITSERVER=undefined
+
+# if the file exists, assume another process started it, and will clean it up
+# when it's done
+if [ -s $LFS_URL_FILE ]; then
+  SHUTDOWN_LFS=no
+else
+  setup
+fi
+
+GITSERVER=$(cat "$LFS_URL_FILE")
+cd "$TRASHDIR"
+
+# Mark the beginning of a test. A subshell should immediately follow this
+# statement.
+begin_test () {
+    test_status=$?
+    [ -n "$test_description" ] && end_test $test_status
+    unset test_status
+
+    tests=$(( tests + 1 ))
+    test_description="$1"
+
+    exec 3>&1 4>&2
+    out="$TRASHDIR/out"
+    err="$TRASHDIR/err"
+    exec 1>"$out" 2>"$err"
+
+    # allow the subshell to exit non-zero without exiting this process
+    set -x +e
+}
+
+# Mark the end of a test.
+end_test () {
+    test_status="${1:-$?}"
+    set +x -e
+    exec 1>&3 2>&4
+
+    if [ "$test_status" -eq 0 ]; then
+        printf "test: %-60s OK\n" "$test_description ..."
+    else
+        failures=$(( failures + 1 ))
+        printf "test: %-60s FAILED\n" "$test_description ..."
+        (
+            echo "-- stdout --"
+            sed 's/^/    /' <"$TRASHDIR/out"
+            echo "-- stderr --"
+            grep -v -e '^\+ end_test' -e '^+ set +x' <"$TRASHDIR/err" |
+                sed 's/^/    /'
+        ) 1>&2
+    fi
+    unset test_description
+}