Support chunked Transfer-Encoding

Fixes github/git-lfs#385

lfs/client.go

@@ -302,7 +302,11 @@ func UploadObject(o *objectResource, cb CopyCallback) *WrappedError {
 	if len(req.Header.Get("Content-Type")) == 0 {
 		req.Header.Set("Content-Type", "application/octet-stream")
 	}
-	req.Header.Set("Content-Length", strconv.FormatInt(o.Size, 10))
+	if req.Header.Get("Transfer-Encoding") == "chunked" {
+		req.TransferEncoding = []string{"chunked"}
+	} else {
+		req.Header.Set("Content-Length", strconv.FormatInt(o.Size, 10))
+	}
 	req.ContentLength = o.Size
 
 	req.Body = ioutil.NopCloser(reader)

test/cmd/lfstest-gitserver.go

@@ -129,11 +129,16 @@ func lfsPostHandler(w http.ResponseWriter, r *http.Request) {
 		Size: obj.Size,
 		Links: map[string]lfsLink{
 			"upload": lfsLink{
-				Href: server.URL + "/storage/" + obj.Oid,
+				Href:   server.URL + "/storage/" + obj.Oid,
+				Header: map[string]string{},
 			},
 		},
 	}
 
+	if testingChunkedTransferEncoding(r) {
+		res.Links["upload"].Header["Transfer-Encoding"] = "chunked"
+	}
+
 	by, err := json.Marshal(res)
 	if err != nil {
 		log.Fatal(err)
@@ -204,17 +209,23 @@ func lfsBatchHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	res := []lfsObject{}
+	testingChunked := testingChunkedTransferEncoding(r)
 	for _, obj := range objs.Objects {
 		o := lfsObject{
 			Oid:  obj.Oid,
 			Size: obj.Size,
 			Links: map[string]lfsLink{
 				"upload": lfsLink{
-					Href: server.URL + "/storage/" + obj.Oid,
+					Href:   server.URL + "/storage/" + obj.Oid,
+					Header: map[string]string{},
 				},
 			},
 		}
 
+		if testingChunked {
+			o.Links["upload"].Header["Transfer-Encoding"] = "chunked"
+		}
+
 		res = append(res, o)
 	}
 
@@ -237,6 +248,19 @@ func storageHandler(w http.ResponseWriter, r *http.Request) {
 	log.Printf("storage %s %s\n", r.Method, r.URL)
 	switch r.Method {
 	case "PUT":
+		if testingChunkedTransferEncoding(r) {
+			valid := false
+			for _, value := range r.TransferEncoding {
+				if value == "chunked" {
+					valid = true
+					break
+				}
+			}
+			if !valid {
+				log.Fatal("Chunked transfer encoding expected")
+			}
+		}
+
 		hash := sha256.New()
 		buf := &bytes.Buffer{}
 		io.Copy(io.MultiWriter(hash, buf), r.Body)
@@ -325,3 +349,7 @@ func redirect307Handler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Location", redirectTo)
 	w.WriteHeader(307)
 }
+
+func testingChunkedTransferEncoding(r *http.Request) bool {
+	return strings.HasPrefix(r.URL.String(), "/test-chunked-transfer-encoding")
+}

test/test-chunked-transfer-encoding-batched.sh

@@ -0,0 +1,67 @@
+#!/bin/sh
+# This is a sample Git LFS test.  See test/README.md and testhelpers.sh for
+# more documentation.
+
+. "test/testlib.sh"
+
+begin_test "chunked transfer encoding batched"
+(
+  set -e
+
+  # This initializes a new bare git repository in test/remote.
+  # These remote repositories are global to every test, so keep the names
+  # unique.
+  reponame="$(basename "$0" ".sh")"
+  setup_remote_repo "$reponame"
+
+  # Clone the repository from the test Git server.  This is empty, and will be
+  # used to test a "git pull" below. The repo is cloned to $TRASHDIR/clone
+  clone_repo "$reponame" clone
+
+  # Clone the repository again to $TRASHDIR/repo. This will be used to commit
+  # and push objects.
+  clone_repo "$reponame" repo
+
+  # This executes Git LFS from the local repo that was just cloned.
+  git lfs track "*.dat" 2>&1 | tee track.log
+  grep "Tracking \*.dat" track.log
+
+  contents="a"
+  contents_oid=$(printf "$contents" | shasum -a 256 | cut -f 1 -d " ")
+
+  printf "$contents" > a.dat
+  git add a.dat
+  git add .gitattributes
+  git commit -m "add a.dat" 2>&1 | tee commit.log
+  grep "master (root-commit)" commit.log
+  grep "2 files changed" commit.log
+  grep "create mode 100644 a.dat" commit.log
+  grep "create mode 100644 .gitattributes" commit.log
+
+  [ "a" = "$(cat a.dat)" ]
+
+  # This is a small shell function that runs several git commands together.
+  assert_pointer "master" "a.dat" "$contents_oid" 1
+
+  refute_server_object "$contents_oid"
+
+  # Ensure batch transfer is turned on for this repo
+  git config --add --local lfs.batch true
+
+  # This pushes to the remote repository set up at the top of the test.
+  git push origin master 2>&1 | tee push.log
+  grep "(1 of 1 files)" push.log
+  grep "master -> master" push.log
+
+  assert_server_object "$contents_oid" "$contents"
+
+  # change to the clone's working directory
+  cd ../clone
+
+  git pull 2>&1 | grep "Downloading a.dat (1 B)"
+
+  [ "a" = "$(cat a.dat)" ]
+
+  assert_pointer "master" "a.dat" "$contents_oid" 1
+)
+end_test

test/test-chunked-transfer-encoding.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+# This is a sample Git LFS test.  See test/README.md and testhelpers.sh for
+# more documentation.
+
+. "test/testlib.sh"
+
+begin_test "chunked transfer encoding"
+(
+  set -e
+
+  # This initializes a new bare git repository in test/remote.
+  # These remote repositories are global to every test, so keep the names
+  # unique.
+  reponame="$(basename "$0" ".sh")"
+  setup_remote_repo "$reponame"
+
+  # Clone the repository from the test Git server.  This is empty, and will be
+  # used to test a "git pull" below. The repo is cloned to $TRASHDIR/clone
+  clone_repo "$reponame" clone
+
+  # Clone the repository again to $TRASHDIR/repo. This will be used to commit
+  # and push objects.
+  clone_repo "$reponame" repo
+
+  # This executes Git LFS from the local repo that was just cloned.
+  git lfs track "*.dat" 2>&1 | tee track.log
+  grep "Tracking \*.dat" track.log
+
+  contents="a"
+  contents_oid=$(printf "$contents" | shasum -a 256 | cut -f 1 -d " ")
+
+  # Regular Git commands can be used.
+  printf "$contents" > a.dat
+  git add a.dat
+  git add .gitattributes
+  git commit -m "add a.dat" 2>&1 | tee commit.log
+  grep "master (root-commit)" commit.log
+  grep "2 files changed" commit.log
+  grep "create mode 100644 a.dat" commit.log
+  grep "create mode 100644 .gitattributes" commit.log
+
+  [ "a" = "$(cat a.dat)" ]
+
+  # This is a small shell function that runs several git commands together.
+  assert_pointer "master" "a.dat" "$contents_oid" 1
+
+  refute_server_object "$contents_oid"
+
+  # This pushes to the remote repository set up at the top of the test.
+  git push origin master 2>&1 | tee push.log
+  grep "(1 of 1 files)" push.log
+  grep "master -> master" push.log
+
+  assert_server_object "$contents_oid" "$contents"
+
+  # change to the clone's working directory
+  cd ../clone
+
+  git pull 2>&1 | grep "Downloading a.dat (1 B)"
+
+  [ "a" = "$(cat a.dat)" ]
+
+  assert_pointer "master" "a.dat" "$contents_oid" 1
+)
+end_test