Merge pull request #1279 from github/experimental/transfer-features-p2

Enhanced transfers: part 2

api/api.go

@@ -20,48 +20,49 @@ import (
 // BatchOrLegacy calls the Batch API and falls back on the Legacy API
 // This is for simplicity, legacy route is not most optimal (serial)
 // TODO LEGACY API: remove when legacy API removed
-func BatchOrLegacy(objects []*ObjectResource, operation string) ([]*ObjectResource, error) {
+func BatchOrLegacy(objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
 	if !config.Config.BatchTransfer() {
-		return Legacy(objects, operation)
+		objs, err := Legacy(objects, operation)
+		return objs, "", err
 	}
-	objs, err := Batch(objects, operation)
+	objs, adapterName, err := Batch(objects, operation, transferAdapters)
 	if err != nil {
 		if errutil.IsNotImplementedError(err) {
 			git.Config.SetLocal("", "lfs.batch", "false")
-			return Legacy(objects, operation)
+			objs, err := Legacy(objects, operation)
+			return objs, "", err
 		}
-		return nil, err
+		return nil, "", err
 	}
-	return objs, nil
+	return objs, adapterName, nil
 }
 
-func BatchOrLegacySingle(inobj *ObjectResource, operation string) (*ObjectResource, error) {
+func BatchOrLegacySingle(inobj *ObjectResource, operation string, transferAdapters []string) (obj *ObjectResource, transferAdapter string, e error) {
-	objs, err := BatchOrLegacy([]*ObjectResource{inobj}, operation)
+	objs, adapterName, err := BatchOrLegacy([]*ObjectResource{inobj}, operation, transferAdapters)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	if len(objs) > 0 {
-		return objs[0], nil
+		return objs[0], adapterName, nil
 	}
-	return nil, fmt.Errorf("Object not found")
+	return nil, "", fmt.Errorf("Object not found")
 }
 
 // Batch calls the batch API and returns object results
-func Batch(objects []*ObjectResource, operation string) ([]*ObjectResource, error) {
+func Batch(objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
 	if len(objects) == 0 {
-		return nil, nil
+		return nil, "", nil
 	}
 
-	o := map[string]interface{}{"objects": objects, "operation": operation}
+	o := &batchRequest{Operation: operation, Objects: objects, TransferAdapterNames: transferAdapters}
-
 	by, err := json.Marshal(o)
 	if err != nil {
-		return nil, errutil.Error(err)
+		return nil, "", errutil.Error(err)
 	}
 
 	req, err := NewBatchRequest(operation)
 	if err != nil {
-		return nil, errutil.Error(err)
+		return nil, "", errutil.Error(err)
 	}
 
 	req.Header.Set("Content-Type", MediaType)
@@ -71,39 +72,39 @@ func Batch(objects []*ObjectResource, operation string) ([]*ObjectResource, erro
 
 	tracerx.Printf("api: batch %d files", len(objects))
 
-	res, objs, err := DoBatchRequest(req)
+	res, bresp, err := DoBatchRequest(req)
 
 	if err != nil {
 
 		if res == nil {
-			return nil, errutil.NewRetriableError(err)
+			return nil, "", errutil.NewRetriableError(err)
 		}
 
 		if res.StatusCode == 0 {
-			return nil, errutil.NewRetriableError(err)
+			return nil, "", errutil.NewRetriableError(err)
 		}
 
 		if errutil.IsAuthError(err) {
 			httputil.SetAuthType(req, res)
-			return Batch(objects, operation)
+			return Batch(objects, operation, transferAdapters)
 		}
 
 		switch res.StatusCode {
 		case 404, 410:
 			tracerx.Printf("api: batch not implemented: %d", res.StatusCode)
-			return nil, errutil.NewNotImplementedError(nil)
+			return nil, "", errutil.NewNotImplementedError(nil)
 		}
 
 		tracerx.Printf("api error: %s", err)
-		return nil, errutil.Error(err)
+		return nil, "", errutil.Error(err)
 	}
 	httputil.LogTransfer("lfs.batch", res)
 
 	if res.StatusCode != 200 {
-		return nil, errutil.Error(fmt.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode))
+		return nil, "", errutil.Error(fmt.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode))
 	}
 
-	return objs, nil
+	return bresp.Objects, bresp.TransferAdapterName, nil
 }
 
 // Legacy calls the legacy API serially and returns ObjectResources

api/download_test.go

@@ -77,7 +77,7 @@ func TestSuccessfulDownload(t *testing.T) {
 	config.Config.SetConfig("lfs.batch", "false")
 	config.Config.SetConfig("lfs.url", server.URL+"/media")
 
-	obj, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download")
+	obj, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
 	if err != nil {
 		if isDockerConnectionError(err) {
 			return
@@ -184,7 +184,7 @@ func TestSuccessfulDownloadWithRedirects(t *testing.T) {
 	config.Config.SetConfig("lfs.url", server.URL+"/redirect")
 
 	for _, redirect := range redirectCodes {
-		obj, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download")
+		obj, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
 		if err != nil {
 			if isDockerConnectionError(err) {
 				return
@@ -260,7 +260,7 @@ func TestSuccessfulDownloadWithAuthorization(t *testing.T) {
 	defer config.Config.ResetConfig()
 	config.Config.SetConfig("lfs.batch", "false")
 	config.Config.SetConfig("lfs.url", server.URL+"/media")
-	obj, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download")
+	obj, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
 	if err != nil {
 		if isDockerConnectionError(err) {
 			return
@@ -294,7 +294,7 @@ func TestDownloadAPIError(t *testing.T) {
 	defer config.Config.ResetConfig()
 	config.Config.SetConfig("lfs.batch", "false")
 	config.Config.SetConfig("lfs.url", server.URL+"/media")
-	_, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download")
+	_, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
 	if err == nil {
 		t.Fatal("no error?")
 	}

api/upload_test.go

@@ -111,7 +111,7 @@ func TestExistingUpload(t *testing.T) {
 
 	oid := filepath.Base(oidPath)
 	stat, _ := os.Stat(oidPath)
-	o, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload")
+	o, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
 	if err != nil {
 		if isDockerConnectionError(err) {
 			return
@@ -237,7 +237,7 @@ func TestUploadWithRedirect(t *testing.T) {
 
 	oid := filepath.Base(oidPath)
 	stat, _ := os.Stat(oidPath)
-	o, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload")
+	o, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
 	if err != nil {
 		if isDockerConnectionError(err) {
 			return
@@ -379,7 +379,7 @@ func TestSuccessfulUploadWithVerify(t *testing.T) {
 
 	oid := filepath.Base(oidPath)
 	stat, _ := os.Stat(oidPath)
-	o, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload")
+	o, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
 	if err != nil {
 		if isDockerConnectionError(err) {
 			return
@@ -431,7 +431,7 @@ func TestUploadApiError(t *testing.T) {
 
 	oid := filepath.Base(oidPath)
 	stat, _ := os.Stat(oidPath)
-	_, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload")
+	_, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
 	if err == nil {
 		t.Fatal(err)
 	}
@@ -549,7 +549,7 @@ func TestUploadVerifyError(t *testing.T) {
 
 	oid := filepath.Base(oidPath)
 	stat, _ := os.Stat(oidPath)
-	o, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload")
+	o, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
 	if err != nil {
 		if isDockerConnectionError(err) {
 			return

api/v1.go

@@ -36,11 +36,21 @@ func DoLegacyRequest(req *http.Request) (*http.Response, *ObjectResource, error)
 	return res, obj, nil
 }
 
+type batchRequest struct {
+	TransferAdapterNames []string          `json:"transfers"`
+	Operation            string            `json:"operation"`
+	Objects              []*ObjectResource `json:"objects"`
+}
+type batchResponse struct {
+	TransferAdapterName string            `json:"transfer"`
+	Objects             []*ObjectResource `json:"objects"`
+}
+
 // doApiBatchRequest runs the request to the LFS batch API. If the API returns a
 // 401, the repo will be marked as having private access and the request will be
 // re-run. When the repo is marked as having private access, credentials will
 // be retrieved.
-func DoBatchRequest(req *http.Request) (*http.Response, []*ObjectResource, error) {
+func DoBatchRequest(req *http.Request) (*http.Response, *batchResponse, error) {
 	res, err := DoRequest(req, config.Config.PrivateAccess(auth.GetOperationForRequest(req)))
 
 	if err != nil {
@@ -50,14 +60,14 @@ func DoBatchRequest(req *http.Request) (*http.Response, []*ObjectResource, error
 		return res, nil, err
 	}
 
-	var objs map[string][]*ObjectResource
+	resp := &batchResponse{}
-	err = httputil.DecodeResponse(res, &objs)
+	err = httputil.DecodeResponse(res, resp)
 
 	if err != nil {
 		httputil.SetErrorResponseContext(err, res)
 	}
 
-	return res, objs["objects"], err
+	return res, resp, err
 }
 
 // DoRequest runs a request to the LFS API, without parsing the response

docs/api/README.md

@@ -7,12 +7,12 @@ flow might look like:
 push`) objects.
 2. The client contacts the Git LFS API to get information about transferring
 the objects.
-3. The client then transfers the objects through the storage API.
+3. The client then transfers the objects through the transfer API.
 
 ## HTTP API
 
 The Git LFS HTTP API is responsible for authenticating the user requests, and
-returning the proper info for the Git LFS client to use the storage API. By
+returning the proper info for the Git LFS client to use the transfer API. By
 default, API endpoint is based on the current Git remote. For example:
 
 ```
@@ -26,11 +26,17 @@ Git LFS endpoint: https://git-server.com/user/repo.git/info/lfs
 The [specification](/docs/spec.md) describes how clients can configure the Git LFS
 API endpoint manually.
 
-The [original v1 API][v1] is used for Git LFS v0.5.x. An experimental [v1
+The [legacy v1 API][legacy] was used for Git LFS v0.5.x. From 0.6.x the 
-batch API][batch] is in the works for v0.6.x.
+[batch API][batch] should always be used where available. 
+
+[legacy]: ./v1/http-v1-legacy.md
+[batch]: ./v1/http-v1-batch.md
+
+From v1.3 there are [optional extensions to the batch API][batch v1.3] for more 
+flexible transfers.
+
+[batch v1.3]: ./v1.3/http-v1.3-batch.md
 
-[v1]: ./http-v1-original.md
-[batch]: ./http-v1-batch.md
 
 ### Authentication
 
@@ -81,43 +87,19 @@ HTTPS is strongly encouraged for all production Git LFS servers.
 If your Git LFS server authenticates with NTLM then you must provide your credentials to `git-credential`
 in the form `username:DOMAIN\user password:password`.
 
-### Hypermedia
+## Transfer API
 
-The Git LFS API uses hypermedia hints to instruct the client what to do next.
+The transfer API is a generic API for directly uploading and downloading objects.
-These links are included in a `_links` property.  Possible relations for objects
-include:
-
-* `self` - This points to the object's canonical API URL.
-* `download` - Follow this link with a GET and the optional header values to
-download the object content.
-* `upload` - Upload the object content to this link with a PUT.
-* `verify` - Optional link for the client to POST after an upload.  If
-included, the client assumes this step is required after uploading an object.
-See the "Verification" section below for more.
-
-Link relations specify the `href`, and optionally a collection of header values
-to set for the request.  These are optional, and depend on the backing object
-store that the Git LFS API is using.  
-
-The Git LFS client will automatically send the same credentials to the followed
-link relation as Basic Authentication IF:
-
-* The url scheme, host, and port all match the Git LFS API endpoint's.
-* The link relation does not specify an Authorization header.
-
-If the host name is different, the Git LFS API needs to send enough information
-through the href query or header values to authenticate the request.
-
-The Git LFS client expects a 200 or 201 response from these hypermedia requests.
-Any other response code is treated as an error.
-
-## Storage API
-
-The Storage API is a generic API for directly uploading and downloading objects.
 Git LFS servers can offload object storage to cloud services like S3, or
 implemented natively in the Git LFS server. The only requirement is that
 hypermedia objects from the Git LFS API return the correct headers so clients
-can access the storage API properly.
+can access the transfer API properly.
+
+As of v1.3 there can be multiple ways files can be uploaded or downloaded, see
+the [v1.3 API doc](v1.3/http-v1.3-batch.md) for details. The following section
+describes the basic transfer method which is the default.
+
+### The basic transfer API
 
 The client downloads objects through individual GET requests. The URL and any
 special headers are provided by  a "download" hypermedia link:
@@ -125,7 +107,7 @@ special headers are provided by  a "download" hypermedia link:
 ```
 # the hypermedia object from the Git LFS API
 # {
-#   "_links": {
+#   "actions": {
 #     "download": {
 #       "href": "https://storage-server.com/OID",
 #       "header": {
@@ -152,7 +134,7 @@ are provided by an "upload" hypermedia link:
 ```
 # the hypermedia object from the Git LFS API
 # {
-#   "_links": {
+#   "actions": {
 #     "upload": {
 #       "href": "https://storage-server.com/OID",
 #       "header": {

docs/api/v1.3/http-v1.3-batch-request-schema.json

@@ -0,0 +1,33 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "title": "Git LFS HTTPS Batch API v1.3 Request",
+  "type": "object",
+  "properties": {
+    "transfers": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+    },
+    "operation": {
+      "type": "string"
+    },
+    "objects": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "oid": {
+            "type": "string"
+          },
+          "size": {
+            "type": "number"
+          }
+        },
+        "required": ["oid", "size"],
+        "additionalProperties": false
+      }
+    }
+  },
+  "required": ["objects", "operation"]
+}

docs/api/v1.3/http-v1.3-batch-response-schema.json

@@ -0,0 +1,79 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "title": "Git LFS HTTPS Batch API v1.3 Response",
+  "type": "object",
+
+  "definitions": {
+    "action": {
+      "type": "object",
+      "properties": {
+        "href": {
+          "type": "string"
+        },
+        "header": {
+          "type": "object",
+          "additionalProperties": true
+        },
+        "expires_at": {
+          "type": "string"
+        }
+      },
+      "required": ["href"],
+      "additionalProperties": false
+    }
+  },
+
+  "properties": {
+    "transfer": {
+      "type": "string"
+    },
+    "objects": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "oid": {
+            "type": "string"
+          },
+          "size": {
+            "type": "number"
+          },
+          "actions": {
+            "type": "object",
+            "properties": {
+              "download": { "$ref": "#/definitions/action" },
+              "upload": { "$ref": "#/definitions/action" },
+              "verify": { "$ref": "#/definitions/action" }
+            },
+            "additionalProperties": false
+          },
+          "error": {
+            "type": "object",
+            "properties": {
+              "code": {
+                "type": "number"
+              },
+              "message": {
+                "type": "string"
+              }
+            },
+            "required": ["code", "message"],
+            "additionalProperties": false
+          }
+        },
+        "required": ["oid", "size"],
+        "additionalProperties": false
+      }
+    },
+    "message": {
+      "type": "string"
+    },
+    "request_id": {
+      "type": "string"
+    },
+    "documentation_url": {
+      "type": "string"
+    },
+  },
+  "required": ["objects"]
+}

docs/api/v1.3/http-v1.3-batch.md

@@ -0,0 +1,98 @@
+# Git LFS v1.3 Batch API
+
+The Git LFS Batch API extends the [batch v1 API](../v1/http-v1-batch.md), adding 
+optional fields to the request and response to negotiate transfer methods. 
+
+Only the differences from the v1 API will be listed here, everything else is
+unchanged.
+
+## POST /objects/batch
+### Request changes
+
+The v1.3 request adds an additional optional top-level field, `transfers`, 
+which is an array of strings naming the transfer methods this client supports.
+The transfer methods are in decreasing order of preference.
+
+The default transfer method which simply uploads and downloads using simple HTTP 
+`PUT` and `GET`, named "basic", is always supported and is implied.
+
+Example request:
+
+```
+> POST https://git-lfs-server.com/objects/batch HTTP/1.1
+> Accept: application/vnd.git-lfs+json
+> Content-Type: application/vnd.git-lfs+json
+> Authorization: Basic ... (if authentication is needed)
+>
+> {
+>   "operation": "upload",
+>   "transfers": [ "tus.io", "basic" ],
+>   "objects": [
+>     {
+>       "oid": "1111111",
+>       "size": 123
+>     }
+>   ]
+> }
+>
+```
+
+In the example above `"basic"` is included for illustration but is actually
+unnecessary since it is always the fallback. The client is indicating that it is
+able to upload using the resumable `"tus.io"` method, should the server support
+that. The server may include a chosen method in the response, which must be
+one of those listed, or `"basic"`.
+
+### Response changes
+
+If the server understands the new optional `transfers` field in the request, it
+should determine which of the named transfer methods it also supports, and
+include the chosen one in the response in the new `transfer` field. If only
+`"basic"` is supported, the field is optional since that is the default.
+
+If the server supports more than one of the named transfer methods, it should
+pick the first one it supports, since the client will list them in order of
+preference.
+
+Example response to the previous request if the server also supports `tus.io`:
+
+```
+< HTTP/1.1 200 Ok
+< Content-Type: application/vnd.git-lfs+json
+<
+< {
+<   "transfer": "tus.io",
+<   "objects": [
+<     {
+<       "oid": "1111111",
+<       "size": 123,
+<       "actions": {
+<         "upload": {
+<           "href": "https://some-tus-io-upload.com",
+<           "header": {
+<             "Key": "value"
+<           }
+<         },
+<         "verify": {
+<           "href": "https://some-callback.com",
+<           "header": {
+<             "Key": "value"
+<           }
+<         }
+<       }
+>     }
+<   ]
+< }
+```
+
+Apart from naming the chosen transfer method in `transfer`, the server should
+also return upload / download links in the `href` field which are compatible 
+with the method chosen. If the server supports more than one method (and it's
+advisable that the server implement at least `"basic` in all cases in addition 
+to more sophisticated methods, to support older clients), the `href` is likely 
+to be different for each. 
+
+## Updated schemas
+
+* [Batch request](./http-v1.3-batch-request-schema.json)
+* [Batch response](./http-v1.3-batch-response-schema.json)

docs/api/v1/http-v1-batch-request-schema.json

@@ -23,6 +23,5 @@
       }
     }
   },
-  "required": ["objects", "operation"],
+  "required": ["objects", "operation"]
-  "additionalProperties": false
 }

docs/api/v1/http-v1-batch-response-schema.json

@@ -72,6 +72,5 @@
       "type": "string"
     },
   },
-  "required": ["objects"],
+  "required": ["objects"]
-  "additionalProperties": false
 }

docs/api/v1/http-v1-batch.md

@@ -1,12 +1,12 @@
 # Git LFS v1 Batch API
 
-The Git LFS Batch API works like the [original v1 API][v1], but uses a single
+The Git LFS Batch API works like the [legacy v1 API][v1], but uses a single
 endpoint that accepts multiple OIDs. All requests should have the following:
 
     Accept: application/vnd.git-lfs+json
     Content-Type: application/vnd.git-lfs+json
 
-[v1]: ./http-v1-original.md
+[v1]: ./http-v1-legacy.md
 
 This is a newer API introduced in Git LFS v0.5.2, and made the default in
 Git LFS v0.6.0. The client automatically detects if the server does not
@@ -21,7 +21,7 @@ manually through the Git config:
 
 ## Authentication
 
-The Batch API authenticates the same as the original v1 API with one exception:
+The Batch API authenticates the same as the legacy v1 API with one exception:
 The client will attempt to make requests without any authentication. This
 slight change allows anonymous access to public Git LFS objects. The client
 stores the result of this in the `lfs.<url>.access` config setting, where <url>

docs/api/v1/http-v1-legacy.md

@@ -1,6 +1,6 @@
-# Git LFS v1 Original API
+# Git LFS v1 Legacy API
 
-This describes the original API for Git LFS v0.5.x. It's already deprecated by
+This describes the legacy API for Git LFS v0.5.x. It's already deprecated by
 the [batch API][batch].  All requests should have:
 
     Accept: application/vnd.git-lfs+json

lfs/download_queue.go

@@ -47,10 +47,10 @@ func NewDownloadable(p *WrappedPointer) *Downloadable {
 // NewDownloadCheckQueue builds a checking queue, checks that objects are there but doesn't download
 func NewDownloadCheckQueue(files int, size int64) *TransferQueue {
 	// Always dry run
-	return newTransferQueue(files, size, true, transfer.NewDownloadAdapter(transfer.BasicAdapterName))
+	return newTransferQueue(files, size, true, transfer.Download)
 }
 
 // NewDownloadQueue builds a DownloadQueue, allowing concurrent downloads.
 func NewDownloadQueue(files int, size int64, dryRun bool) *TransferQueue {
-	return newTransferQueue(files, size, dryRun, transfer.NewDownloadAdapter(transfer.BasicAdapterName))
+	return newTransferQueue(files, size, dryRun, transfer.Download)
 }

lfs/pointer_smudge.go

@@ -76,7 +76,8 @@ func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download
 func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, cb progress.CopyCallback) error {
 	fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, pb.FormatBytes(ptr.Size))
 
-	obj, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: ptr.Oid, Size: ptr.Size}, "download")
+	xfers := transfer.GetDownloadAdapterNames()
+	obj, adapterName, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: ptr.Oid, Size: ptr.Size}, "download", xfers)
 	if err != nil {
 		return errutil.Errorf(err, "Error downloading %s: %s", filepath.Base(mediafile), err)
 	}
@@ -85,7 +86,7 @@ func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string,
 		ptr.Size = obj.Size
 	}
 
-	adapter := transfer.NewDownloadAdapter(transfer.BasicAdapterName)
+	adapter := transfer.NewDownloadAdapter(adapterName)
 	var tcb transfer.TransferProgressCallback
 	if cb != nil {
 		tcb = func(name string, totalSize, readSoFar int64, readSinceLast int) error {

lfs/transfer_queue.go

@@ -32,6 +32,7 @@ type Transferable interface {
 // including calling the API, passing the actual transfer request to transfer
 // adapters, and dealing with progress, errors and retries
 type TransferQueue struct {
+	direction         transfer.Direction
 	adapter           transfer.TransferAdapter
 	adapterInProgress bool
 	adapterResultChan chan transfer.TransferResult
@@ -55,9 +56,9 @@ type TransferQueue struct {
 }
 
 // newTransferQueue builds a TransferQueue, direction and underlying mechanism determined by adapter
-func newTransferQueue(files int, size int64, dryRun bool, adapter transfer.TransferAdapter) *TransferQueue {
+func newTransferQueue(files int, size int64, dryRun bool, dir transfer.Direction) *TransferQueue {
 	q := &TransferQueue{
-		adapter:       adapter,
+		direction:     dir,
 		dryRun:        dryRun,
 		meter:         progress.NewProgressMeter(files, size, dryRun, config.Config.Getenv("GIT_LFS_PROGRESS")),
 		apic:          make(chan Transferable, batchSize),
@@ -91,6 +92,32 @@ func (q *TransferQueue) Add(t Transferable) {
 	q.apic <- t
 }
 
+func (q *TransferQueue) useAdapter(name string) {
+	q.adapterInitMutex.Lock()
+	defer q.adapterInitMutex.Unlock()
+
+	if q.adapter != nil {
+		if q.adapter.Name() == name {
+			// re-use, this is the normal path
+			return
+		}
+		// If the adapter we're using isn't the same as the one we've been
+		// told to use now, must wait for the current one to finish then switch
+		// This will probably never happen but is just in case server starts
+		// changing adapter support in between batches
+		q.finishAdapter()
+	}
+	q.adapter = transfer.NewAdapterOrDefault(name, q.direction)
+}
+
+func (q *TransferQueue) finishAdapter() {
+	if q.adapterInProgress {
+		q.adapter.End()
+		q.adapterInProgress = false
+		q.adapter = nil
+	}
+}
+
 func (q *TransferQueue) addToAdapter(t Transferable) {
 
 	tr := transfer.NewTransfer(t.Name(), t.Object(), t.Path())
@@ -110,7 +137,7 @@ func (q *TransferQueue) Skip(size int64) {
 }
 
 func (q *TransferQueue) transferKind() string {
-	if q.adapter.Direction() == transfer.Download {
+	if q.direction == transfer.Download {
 		return "download"
 	} else {
 		return "upload"
@@ -202,10 +229,7 @@ func (q *TransferQueue) Wait() {
 	atomic.StoreUint32(&q.retrying, 0)
 
 	close(q.apic)
-	if q.adapterInProgress {
+	q.finishAdapter()
-		q.adapter.End()
-		q.adapterInProgress = false
-	}
 	close(q.errorc)
 
 	for _, watcher := range q.watchers {
@@ -250,6 +274,8 @@ func (q *TransferQueue) individualApiRoutine(apiWaiter chan interface{}) {
 			}
 		}
 
+		// Legacy API has no support for anything but basic transfer adapter
+		q.useAdapter(transfer.BasicAdapterName)
 		if obj != nil {
 			t.SetObject(obj)
 			q.meter.Add(t.Name())
@@ -292,6 +318,8 @@ func (q *TransferQueue) legacyFallback(failedBatch []interface{}) {
 func (q *TransferQueue) batchApiRoutine() {
 	var startProgress sync.Once
 
+	transferAdapterNames := transfer.GetAdapterNames(q.direction)
+
 	for {
 		batch := q.batcher.Next()
 		if batch == nil {
@@ -306,7 +334,11 @@ func (q *TransferQueue) batchApiRoutine() {
 			transfers = append(transfers, &api.ObjectResource{Oid: t.Oid(), Size: t.Size()})
 		}
 
-		objects, err := api.Batch(transfers, q.transferKind())
+		if len(transfers) == 0 {
+			continue
+		}
+
+		objs, adapterName, err := api.Batch(transfers, q.transferKind(), transferAdapterNames)
 		if err != nil {
 			if errutil.IsNotImplementedError(err) {
 				git.Config.SetLocal("", "lfs.batch", "false")
@@ -327,9 +359,10 @@ func (q *TransferQueue) batchApiRoutine() {
 			continue
 		}
 
+		q.useAdapter(adapterName)
 		startProgress.Do(q.meter.Start)
 
-		for _, o := range objects {
+		for _, o := range objs {
 			if o.Error != nil {
 				q.errorc <- errutil.Errorf(o.Error, "[%v] %v", o.Oid, o.Error.Message)
 				q.Skip(o.Size)

lfs/upload_queue.go

@@ -73,7 +73,7 @@ func NewUploadable(oid, filename string) (*Uploadable, error) {
 
 // NewUploadQueue builds an UploadQueue, allowing `workers` concurrent uploads.
 func NewUploadQueue(files int, size int64, dryRun bool) *TransferQueue {
-	return newTransferQueue(files, size, dryRun, transfer.NewUploadAdapter(transfer.BasicAdapterName))
+	return newTransferQueue(files, size, dryRun, transfer.Upload)
 }
 
 // ensureFile makes sure that the cleanPath exists before pushing it.  If it

test/cmd/lfstest-gitserver.go

@@ -133,8 +133,8 @@ type lfsError struct {
 func lfsHandler(w http.ResponseWriter, r *http.Request) {
 	repo, err := repoFromLfsUrl(r.URL.Path)
 	if err != nil {
-		w.Write([]byte(err.Error()))
 		w.WriteHeader(500)
+		w.Write([]byte(err.Error()))
 		return
 	}
 
@@ -510,15 +510,15 @@ func missingRequiredCreds(w http.ResponseWriter, r *http.Request, repo string) b
 	auth := r.Header.Get("Authorization")
 	user, pass, err := extractAuth(auth)
 	if err != nil {
-		w.Write([]byte(`{"message":"` + err.Error() + `"}`))
 		w.WriteHeader(403)
+		w.Write([]byte(`{"message":"` + err.Error() + `"}`))
 		return true
 	}
 
 	if user != "requirecreds" || pass != "pass" {
 		errmsg := fmt.Sprintf("Got: '%s' => '%s' : '%s'", auth, user, pass)
-		w.Write([]byte(`{"message":"` + errmsg + `"}`))
 		w.WriteHeader(403)
+		w.Write([]byte(`{"message":"` + errmsg + `"}`))
 		return true
 	}
 

test/git-lfs-test-server-api/main.go

@@ -250,7 +250,11 @@ func callBatchApi(op string, objs []TestObject) ([]*api.ObjectResource, error) {
 	for _, o := range objs {
 		apiobjs = append(apiobjs, &api.ObjectResource{Oid: o.Oid, Size: o.Size})
 	}
-	return api.Batch(apiobjs, op)
+	o, _, err := api.Batch(apiobjs, op, []string{"basic"})
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
 }
 
 // Combine 2 slices into one by "randomly" interleaving

transfer/transfer.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	"github.com/github/git-lfs/api"
+	"github.com/rubyist/tracerx"
 )
 
 type Direction int
@@ -140,6 +141,20 @@ func RegisterNewTransferAdapterFunc(name string, dir Direction, f NewTransferAda
 	}
 }
 
+// Create a new adapter by name and direction; default to BasicAdapterName if doesn't exist
+func NewAdapterOrDefault(name string, dir Direction) TransferAdapter {
+	if len(name) == 0 {
+		name = BasicAdapterName
+	}
+
+	a := NewAdapter(name, dir)
+	if a == nil {
+		tracerx.Printf("Defaulting to basic transfer adapter since %q did not exist", name)
+		a = NewAdapter(BasicAdapterName, dir)
+	}
+	return a
+}
+
 // Create a new adapter by name and direction, or nil if doesn't exist
 func NewAdapter(name string, dir Direction) TransferAdapter {
 	funcMutex.Lock()
@@ -158,12 +173,12 @@ func NewAdapter(name string, dir Direction) TransferAdapter {
 	return nil
 }
 
-// Create a new download adapter by name, or nil if doesn't exist
+// Create a new download adapter by name, or BasicAdapterName if doesn't exist
 func NewDownloadAdapter(name string) TransferAdapter {
-	return NewAdapter(name, Download)
+	return NewAdapterOrDefault(name, Download)
 }
 
-// Create a new upload adapter by name, or nil if doesn't exist
+// Create a new upload adapter by name, or BasicAdapterName if doesn't exist
 func NewUploadAdapter(name string) TransferAdapter {
-	return NewAdapter(name, Upload)
+	return NewAdapterOrDefault(name, Upload)
 }