Merge pull request #1279 from github/experimental/transfer-features-p2
Enhanced transfers: part 2
This commit is contained in:
commit
12fe249f2e
51
api/api.go
51
api/api.go
@ -20,48 +20,49 @@ import (
|
||||
// BatchOrLegacy calls the Batch API and falls back on the Legacy API
|
||||
// This is for simplicity, legacy route is not most optimal (serial)
|
||||
// TODO LEGACY API: remove when legacy API removed
|
||||
func BatchOrLegacy(objects []*ObjectResource, operation string) ([]*ObjectResource, error) {
|
||||
func BatchOrLegacy(objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
|
||||
if !config.Config.BatchTransfer() {
|
||||
return Legacy(objects, operation)
|
||||
objs, err := Legacy(objects, operation)
|
||||
return objs, "", err
|
||||
}
|
||||
objs, err := Batch(objects, operation)
|
||||
objs, adapterName, err := Batch(objects, operation, transferAdapters)
|
||||
if err != nil {
|
||||
if errutil.IsNotImplementedError(err) {
|
||||
git.Config.SetLocal("", "lfs.batch", "false")
|
||||
return Legacy(objects, operation)
|
||||
objs, err := Legacy(objects, operation)
|
||||
return objs, "", err
|
||||
}
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
return objs, nil
|
||||
return objs, adapterName, nil
|
||||
}
|
||||
|
||||
func BatchOrLegacySingle(inobj *ObjectResource, operation string) (*ObjectResource, error) {
|
||||
objs, err := BatchOrLegacy([]*ObjectResource{inobj}, operation)
|
||||
func BatchOrLegacySingle(inobj *ObjectResource, operation string, transferAdapters []string) (obj *ObjectResource, transferAdapter string, e error) {
|
||||
objs, adapterName, err := BatchOrLegacy([]*ObjectResource{inobj}, operation, transferAdapters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
if len(objs) > 0 {
|
||||
return objs[0], nil
|
||||
return objs[0], adapterName, nil
|
||||
}
|
||||
return nil, fmt.Errorf("Object not found")
|
||||
return nil, "", fmt.Errorf("Object not found")
|
||||
}
|
||||
|
||||
// Batch calls the batch API and returns object results
|
||||
func Batch(objects []*ObjectResource, operation string) ([]*ObjectResource, error) {
|
||||
func Batch(objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
|
||||
if len(objects) == 0 {
|
||||
return nil, nil
|
||||
return nil, "", nil
|
||||
}
|
||||
|
||||
o := map[string]interface{}{"objects": objects, "operation": operation}
|
||||
|
||||
o := &batchRequest{Operation: operation, Objects: objects, TransferAdapterNames: transferAdapters}
|
||||
by, err := json.Marshal(o)
|
||||
if err != nil {
|
||||
return nil, errutil.Error(err)
|
||||
return nil, "", errutil.Error(err)
|
||||
}
|
||||
|
||||
req, err := NewBatchRequest(operation)
|
||||
if err != nil {
|
||||
return nil, errutil.Error(err)
|
||||
return nil, "", errutil.Error(err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", MediaType)
|
||||
@ -71,39 +72,39 @@ func Batch(objects []*ObjectResource, operation string) ([]*ObjectResource, erro
|
||||
|
||||
tracerx.Printf("api: batch %d files", len(objects))
|
||||
|
||||
res, objs, err := DoBatchRequest(req)
|
||||
res, bresp, err := DoBatchRequest(req)
|
||||
|
||||
if err != nil {
|
||||
|
||||
if res == nil {
|
||||
return nil, errutil.NewRetriableError(err)
|
||||
return nil, "", errutil.NewRetriableError(err)
|
||||
}
|
||||
|
||||
if res.StatusCode == 0 {
|
||||
return nil, errutil.NewRetriableError(err)
|
||||
return nil, "", errutil.NewRetriableError(err)
|
||||
}
|
||||
|
||||
if errutil.IsAuthError(err) {
|
||||
httputil.SetAuthType(req, res)
|
||||
return Batch(objects, operation)
|
||||
return Batch(objects, operation, transferAdapters)
|
||||
}
|
||||
|
||||
switch res.StatusCode {
|
||||
case 404, 410:
|
||||
tracerx.Printf("api: batch not implemented: %d", res.StatusCode)
|
||||
return nil, errutil.NewNotImplementedError(nil)
|
||||
return nil, "", errutil.NewNotImplementedError(nil)
|
||||
}
|
||||
|
||||
tracerx.Printf("api error: %s", err)
|
||||
return nil, errutil.Error(err)
|
||||
return nil, "", errutil.Error(err)
|
||||
}
|
||||
httputil.LogTransfer("lfs.batch", res)
|
||||
|
||||
if res.StatusCode != 200 {
|
||||
return nil, errutil.Error(fmt.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode))
|
||||
return nil, "", errutil.Error(fmt.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode))
|
||||
}
|
||||
|
||||
return objs, nil
|
||||
return bresp.Objects, bresp.TransferAdapterName, nil
|
||||
}
|
||||
|
||||
// Legacy calls the legacy API serially and returns ObjectResources
|
||||
|
@ -77,7 +77,7 @@ func TestSuccessfulDownload(t *testing.T) {
|
||||
config.Config.SetConfig("lfs.batch", "false")
|
||||
config.Config.SetConfig("lfs.url", server.URL+"/media")
|
||||
|
||||
obj, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download")
|
||||
obj, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
|
||||
if err != nil {
|
||||
if isDockerConnectionError(err) {
|
||||
return
|
||||
@ -184,7 +184,7 @@ func TestSuccessfulDownloadWithRedirects(t *testing.T) {
|
||||
config.Config.SetConfig("lfs.url", server.URL+"/redirect")
|
||||
|
||||
for _, redirect := range redirectCodes {
|
||||
obj, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download")
|
||||
obj, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
|
||||
if err != nil {
|
||||
if isDockerConnectionError(err) {
|
||||
return
|
||||
@ -260,7 +260,7 @@ func TestSuccessfulDownloadWithAuthorization(t *testing.T) {
|
||||
defer config.Config.ResetConfig()
|
||||
config.Config.SetConfig("lfs.batch", "false")
|
||||
config.Config.SetConfig("lfs.url", server.URL+"/media")
|
||||
obj, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download")
|
||||
obj, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
|
||||
if err != nil {
|
||||
if isDockerConnectionError(err) {
|
||||
return
|
||||
@ -294,7 +294,7 @@ func TestDownloadAPIError(t *testing.T) {
|
||||
defer config.Config.ResetConfig()
|
||||
config.Config.SetConfig("lfs.batch", "false")
|
||||
config.Config.SetConfig("lfs.url", server.URL+"/media")
|
||||
_, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download")
|
||||
_, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
|
||||
if err == nil {
|
||||
t.Fatal("no error?")
|
||||
}
|
||||
|
@ -111,7 +111,7 @@ func TestExistingUpload(t *testing.T) {
|
||||
|
||||
oid := filepath.Base(oidPath)
|
||||
stat, _ := os.Stat(oidPath)
|
||||
o, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload")
|
||||
o, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
|
||||
if err != nil {
|
||||
if isDockerConnectionError(err) {
|
||||
return
|
||||
@ -237,7 +237,7 @@ func TestUploadWithRedirect(t *testing.T) {
|
||||
|
||||
oid := filepath.Base(oidPath)
|
||||
stat, _ := os.Stat(oidPath)
|
||||
o, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload")
|
||||
o, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
|
||||
if err != nil {
|
||||
if isDockerConnectionError(err) {
|
||||
return
|
||||
@ -379,7 +379,7 @@ func TestSuccessfulUploadWithVerify(t *testing.T) {
|
||||
|
||||
oid := filepath.Base(oidPath)
|
||||
stat, _ := os.Stat(oidPath)
|
||||
o, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload")
|
||||
o, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
|
||||
if err != nil {
|
||||
if isDockerConnectionError(err) {
|
||||
return
|
||||
@ -431,7 +431,7 @@ func TestUploadApiError(t *testing.T) {
|
||||
|
||||
oid := filepath.Base(oidPath)
|
||||
stat, _ := os.Stat(oidPath)
|
||||
_, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload")
|
||||
_, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
|
||||
if err == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -549,7 +549,7 @@ func TestUploadVerifyError(t *testing.T) {
|
||||
|
||||
oid := filepath.Base(oidPath)
|
||||
stat, _ := os.Stat(oidPath)
|
||||
o, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload")
|
||||
o, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
|
||||
if err != nil {
|
||||
if isDockerConnectionError(err) {
|
||||
return
|
||||
|
18
api/v1.go
18
api/v1.go
@ -36,11 +36,21 @@ func DoLegacyRequest(req *http.Request) (*http.Response, *ObjectResource, error)
|
||||
return res, obj, nil
|
||||
}
|
||||
|
||||
type batchRequest struct {
|
||||
TransferAdapterNames []string `json:"transfers"`
|
||||
Operation string `json:"operation"`
|
||||
Objects []*ObjectResource `json:"objects"`
|
||||
}
|
||||
type batchResponse struct {
|
||||
TransferAdapterName string `json:"transfer"`
|
||||
Objects []*ObjectResource `json:"objects"`
|
||||
}
|
||||
|
||||
// doApiBatchRequest runs the request to the LFS batch API. If the API returns a
|
||||
// 401, the repo will be marked as having private access and the request will be
|
||||
// re-run. When the repo is marked as having private access, credentials will
|
||||
// be retrieved.
|
||||
func DoBatchRequest(req *http.Request) (*http.Response, []*ObjectResource, error) {
|
||||
func DoBatchRequest(req *http.Request) (*http.Response, *batchResponse, error) {
|
||||
res, err := DoRequest(req, config.Config.PrivateAccess(auth.GetOperationForRequest(req)))
|
||||
|
||||
if err != nil {
|
||||
@ -50,14 +60,14 @@ func DoBatchRequest(req *http.Request) (*http.Response, []*ObjectResource, error
|
||||
return res, nil, err
|
||||
}
|
||||
|
||||
var objs map[string][]*ObjectResource
|
||||
err = httputil.DecodeResponse(res, &objs)
|
||||
resp := &batchResponse{}
|
||||
err = httputil.DecodeResponse(res, resp)
|
||||
|
||||
if err != nil {
|
||||
httputil.SetErrorResponseContext(err, res)
|
||||
}
|
||||
|
||||
return res, objs["objects"], err
|
||||
return res, resp, err
|
||||
}
|
||||
|
||||
// DoRequest runs a request to the LFS API, without parsing the response
|
||||
|
@ -7,12 +7,12 @@ flow might look like:
|
||||
push`) objects.
|
||||
2. The client contacts the Git LFS API to get information about transferring
|
||||
the objects.
|
||||
3. The client then transfers the objects through the storage API.
|
||||
3. The client then transfers the objects through the transfer API.
|
||||
|
||||
## HTTP API
|
||||
|
||||
The Git LFS HTTP API is responsible for authenticating the user requests, and
|
||||
returning the proper info for the Git LFS client to use the storage API. By
|
||||
returning the proper info for the Git LFS client to use the transfer API. By
|
||||
default, API endpoint is based on the current Git remote. For example:
|
||||
|
||||
```
|
||||
@ -26,11 +26,17 @@ Git LFS endpoint: https://git-server.com/user/repo.git/info/lfs
|
||||
The [specification](/docs/spec.md) describes how clients can configure the Git LFS
|
||||
API endpoint manually.
|
||||
|
||||
The [original v1 API][v1] is used for Git LFS v0.5.x. An experimental [v1
|
||||
batch API][batch] is in the works for v0.6.x.
|
||||
The [legacy v1 API][legacy] was used for Git LFS v0.5.x. From 0.6.x the
|
||||
[batch API][batch] should always be used where available.
|
||||
|
||||
[legacy]: ./v1/http-v1-legacy.md
|
||||
[batch]: ./v1/http-v1-batch.md
|
||||
|
||||
From v1.3 there are [optional extensions to the batch API][batch v1.3] for more
|
||||
flexible transfers.
|
||||
|
||||
[batch v1.3]: ./v1.3/http-v1.3-batch.md
|
||||
|
||||
[v1]: ./http-v1-original.md
|
||||
[batch]: ./http-v1-batch.md
|
||||
|
||||
### Authentication
|
||||
|
||||
@ -81,43 +87,19 @@ HTTPS is strongly encouraged for all production Git LFS servers.
|
||||
If your Git LFS server authenticates with NTLM then you must provide your credentials to `git-credential`
|
||||
in the form `username:DOMAIN\user password:password`.
|
||||
|
||||
### Hypermedia
|
||||
## Transfer API
|
||||
|
||||
The Git LFS API uses hypermedia hints to instruct the client what to do next.
|
||||
These links are included in a `_links` property. Possible relations for objects
|
||||
include:
|
||||
|
||||
* `self` - This points to the object's canonical API URL.
|
||||
* `download` - Follow this link with a GET and the optional header values to
|
||||
download the object content.
|
||||
* `upload` - Upload the object content to this link with a PUT.
|
||||
* `verify` - Optional link for the client to POST after an upload. If
|
||||
included, the client assumes this step is required after uploading an object.
|
||||
See the "Verification" section below for more.
|
||||
|
||||
Link relations specify the `href`, and optionally a collection of header values
|
||||
to set for the request. These are optional, and depend on the backing object
|
||||
store that the Git LFS API is using.
|
||||
|
||||
The Git LFS client will automatically send the same credentials to the followed
|
||||
link relation as Basic Authentication IF:
|
||||
|
||||
* The url scheme, host, and port all match the Git LFS API endpoint's.
|
||||
* The link relation does not specify an Authorization header.
|
||||
|
||||
If the host name is different, the Git LFS API needs to send enough information
|
||||
through the href query or header values to authenticate the request.
|
||||
|
||||
The Git LFS client expects a 200 or 201 response from these hypermedia requests.
|
||||
Any other response code is treated as an error.
|
||||
|
||||
## Storage API
|
||||
|
||||
The Storage API is a generic API for directly uploading and downloading objects.
|
||||
The transfer API is a generic API for directly uploading and downloading objects.
|
||||
Git LFS servers can offload object storage to cloud services like S3, or
|
||||
implemented natively in the Git LFS server. The only requirement is that
|
||||
hypermedia objects from the Git LFS API return the correct headers so clients
|
||||
can access the storage API properly.
|
||||
can access the transfer API properly.
|
||||
|
||||
As of v1.3 there can be multiple ways files can be uploaded or downloaded, see
|
||||
the [v1.3 API doc](v1.3/http-v1.3-batch.md) for details. The following section
|
||||
describes the basic transfer method which is the default.
|
||||
|
||||
### The basic transfer API
|
||||
|
||||
The client downloads objects through individual GET requests. The URL and any
|
||||
special headers are provided by a "download" hypermedia link:
|
||||
@ -125,7 +107,7 @@ special headers are provided by a "download" hypermedia link:
|
||||
```
|
||||
# the hypermedia object from the Git LFS API
|
||||
# {
|
||||
# "_links": {
|
||||
# "actions": {
|
||||
# "download": {
|
||||
# "href": "https://storage-server.com/OID",
|
||||
# "header": {
|
||||
@ -152,7 +134,7 @@ are provided by an "upload" hypermedia link:
|
||||
```
|
||||
# the hypermedia object from the Git LFS API
|
||||
# {
|
||||
# "_links": {
|
||||
# "actions": {
|
||||
# "upload": {
|
||||
# "href": "https://storage-server.com/OID",
|
||||
# "header": {
|
||||
|
33
docs/api/v1.3/http-v1.3-batch-request-schema.json
Normal file
33
docs/api/v1.3/http-v1.3-batch-request-schema.json
Normal file
@ -0,0 +1,33 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-04/schema",
|
||||
"title": "Git LFS HTTPS Batch API v1.3 Request",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"transfers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
},
|
||||
"operation": {
|
||||
"type": "string"
|
||||
},
|
||||
"objects": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"oid": {
|
||||
"type": "string"
|
||||
},
|
||||
"size": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": ["oid", "size"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["objects", "operation"]
|
||||
}
|
79
docs/api/v1.3/http-v1.3-batch-response-schema.json
Normal file
79
docs/api/v1.3/http-v1.3-batch-response-schema.json
Normal file
@ -0,0 +1,79 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-04/schema",
|
||||
"title": "Git LFS HTTPS Batch API v1.3 Response",
|
||||
"type": "object",
|
||||
|
||||
"definitions": {
|
||||
"action": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"href": {
|
||||
"type": "string"
|
||||
},
|
||||
"header": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
},
|
||||
"expires_at": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["href"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
|
||||
"properties": {
|
||||
"transfer": {
|
||||
"type": "string"
|
||||
},
|
||||
"objects": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"oid": {
|
||||
"type": "string"
|
||||
},
|
||||
"size": {
|
||||
"type": "number"
|
||||
},
|
||||
"actions": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"download": { "$ref": "#/definitions/action" },
|
||||
"upload": { "$ref": "#/definitions/action" },
|
||||
"verify": { "$ref": "#/definitions/action" }
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"error": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {
|
||||
"type": "number"
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["code", "message"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": ["oid", "size"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"request_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"documentation_url": {
|
||||
"type": "string"
|
||||
},
|
||||
},
|
||||
"required": ["objects"]
|
||||
}
|
98
docs/api/v1.3/http-v1.3-batch.md
Normal file
98
docs/api/v1.3/http-v1.3-batch.md
Normal file
@ -0,0 +1,98 @@
|
||||
# Git LFS v1.3 Batch API
|
||||
|
||||
The Git LFS Batch API extends the [batch v1 API](../v1/http-v1-batch.md), adding
|
||||
optional fields to the request and response to negotiate transfer methods.
|
||||
|
||||
Only the differences from the v1 API will be listed here, everything else is
|
||||
unchanged.
|
||||
|
||||
## POST /objects/batch
|
||||
### Request changes
|
||||
|
||||
The v1.3 request adds an additional optional top-level field, `transfers`,
|
||||
which is an array of strings naming the transfer methods this client supports.
|
||||
The transfer methods are in decreasing order of preference.
|
||||
|
||||
The default transfer method which simply uploads and downloads using simple HTTP
|
||||
`PUT` and `GET`, named "basic", is always supported and is implied.
|
||||
|
||||
Example request:
|
||||
|
||||
```
|
||||
> POST https://git-lfs-server.com/objects/batch HTTP/1.1
|
||||
> Accept: application/vnd.git-lfs+json
|
||||
> Content-Type: application/vnd.git-lfs+json
|
||||
> Authorization: Basic ... (if authentication is needed)
|
||||
>
|
||||
> {
|
||||
> "operation": "upload",
|
||||
> "transfers": [ "tus.io", "basic" ],
|
||||
> "objects": [
|
||||
> {
|
||||
> "oid": "1111111",
|
||||
> "size": 123
|
||||
> }
|
||||
> ]
|
||||
> }
|
||||
>
|
||||
```
|
||||
|
||||
In the example above `"basic"` is included for illustration but is actually
|
||||
unnecessary since it is always the fallback. The client is indicating that it is
|
||||
able to upload using the resumable `"tus.io"` method, should the server support
|
||||
that. The server may include a chosen method in the response, which must be
|
||||
one of those listed, or `"basic"`.
|
||||
|
||||
### Response changes
|
||||
|
||||
If the server understands the new optional `transfers` field in the request, it
|
||||
should determine which of the named transfer methods it also supports, and
|
||||
include the chosen one in the response in the new `transfer` field. If only
|
||||
`"basic"` is supported, the field is optional since that is the default.
|
||||
|
||||
If the server supports more than one of the named transfer methods, it should
|
||||
pick the first one it supports, since the client will list them in order of
|
||||
preference.
|
||||
|
||||
Example response to the previous request if the server also supports `tus.io`:
|
||||
|
||||
```
|
||||
< HTTP/1.1 200 Ok
|
||||
< Content-Type: application/vnd.git-lfs+json
|
||||
<
|
||||
< {
|
||||
< "transfer": "tus.io",
|
||||
< "objects": [
|
||||
< {
|
||||
< "oid": "1111111",
|
||||
< "size": 123,
|
||||
< "actions": {
|
||||
< "upload": {
|
||||
< "href": "https://some-tus-io-upload.com",
|
||||
< "header": {
|
||||
< "Key": "value"
|
||||
< }
|
||||
< },
|
||||
< "verify": {
|
||||
< "href": "https://some-callback.com",
|
||||
< "header": {
|
||||
< "Key": "value"
|
||||
< }
|
||||
< }
|
||||
< }
|
||||
> }
|
||||
< ]
|
||||
< }
|
||||
```
|
||||
|
||||
Apart from naming the chosen transfer method in `transfer`, the server should
|
||||
also return upload / download links in the `href` field which are compatible
|
||||
with the method chosen. If the server supports more than one method (and it's
|
||||
advisable that the server implement at least `"basic` in all cases in addition
|
||||
to more sophisticated methods, to support older clients), the `href` is likely
|
||||
to be different for each.
|
||||
|
||||
## Updated schemas
|
||||
|
||||
* [Batch request](./http-v1.3-batch-request-schema.json)
|
||||
* [Batch response](./http-v1.3-batch-response-schema.json)
|
@ -23,6 +23,5 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["objects", "operation"],
|
||||
"additionalProperties": false
|
||||
"required": ["objects", "operation"]
|
||||
}
|
@ -72,6 +72,5 @@
|
||||
"type": "string"
|
||||
},
|
||||
},
|
||||
"required": ["objects"],
|
||||
"additionalProperties": false
|
||||
"required": ["objects"]
|
||||
}
|
@ -1,12 +1,12 @@
|
||||
# Git LFS v1 Batch API
|
||||
|
||||
The Git LFS Batch API works like the [original v1 API][v1], but uses a single
|
||||
The Git LFS Batch API works like the [legacy v1 API][v1], but uses a single
|
||||
endpoint that accepts multiple OIDs. All requests should have the following:
|
||||
|
||||
Accept: application/vnd.git-lfs+json
|
||||
Content-Type: application/vnd.git-lfs+json
|
||||
|
||||
[v1]: ./http-v1-original.md
|
||||
[v1]: ./http-v1-legacy.md
|
||||
|
||||
This is a newer API introduced in Git LFS v0.5.2, and made the default in
|
||||
Git LFS v0.6.0. The client automatically detects if the server does not
|
||||
@ -21,7 +21,7 @@ manually through the Git config:
|
||||
|
||||
## Authentication
|
||||
|
||||
The Batch API authenticates the same as the original v1 API with one exception:
|
||||
The Batch API authenticates the same as the legacy v1 API with one exception:
|
||||
The client will attempt to make requests without any authentication. This
|
||||
slight change allows anonymous access to public Git LFS objects. The client
|
||||
stores the result of this in the `lfs.<url>.access` config setting, where <url>
|
@ -1,6 +1,6 @@
|
||||
# Git LFS v1 Original API
|
||||
# Git LFS v1 Legacy API
|
||||
|
||||
This describes the original API for Git LFS v0.5.x. It's already deprecated by
|
||||
This describes the legacy API for Git LFS v0.5.x. It's already deprecated by
|
||||
the [batch API][batch]. All requests should have:
|
||||
|
||||
Accept: application/vnd.git-lfs+json
|
@ -47,10 +47,10 @@ func NewDownloadable(p *WrappedPointer) *Downloadable {
|
||||
// NewDownloadCheckQueue builds a checking queue, checks that objects are there but doesn't download
|
||||
func NewDownloadCheckQueue(files int, size int64) *TransferQueue {
|
||||
// Always dry run
|
||||
return newTransferQueue(files, size, true, transfer.NewDownloadAdapter(transfer.BasicAdapterName))
|
||||
return newTransferQueue(files, size, true, transfer.Download)
|
||||
}
|
||||
|
||||
// NewDownloadQueue builds a DownloadQueue, allowing concurrent downloads.
|
||||
func NewDownloadQueue(files int, size int64, dryRun bool) *TransferQueue {
|
||||
return newTransferQueue(files, size, dryRun, transfer.NewDownloadAdapter(transfer.BasicAdapterName))
|
||||
return newTransferQueue(files, size, dryRun, transfer.Download)
|
||||
}
|
||||
|
@ -76,7 +76,8 @@ func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download
|
||||
func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, cb progress.CopyCallback) error {
|
||||
fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, pb.FormatBytes(ptr.Size))
|
||||
|
||||
obj, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: ptr.Oid, Size: ptr.Size}, "download")
|
||||
xfers := transfer.GetDownloadAdapterNames()
|
||||
obj, adapterName, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: ptr.Oid, Size: ptr.Size}, "download", xfers)
|
||||
if err != nil {
|
||||
return errutil.Errorf(err, "Error downloading %s: %s", filepath.Base(mediafile), err)
|
||||
}
|
||||
@ -85,7 +86,7 @@ func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string,
|
||||
ptr.Size = obj.Size
|
||||
}
|
||||
|
||||
adapter := transfer.NewDownloadAdapter(transfer.BasicAdapterName)
|
||||
adapter := transfer.NewDownloadAdapter(adapterName)
|
||||
var tcb transfer.TransferProgressCallback
|
||||
if cb != nil {
|
||||
tcb = func(name string, totalSize, readSoFar int64, readSinceLast int) error {
|
||||
|
@ -32,6 +32,7 @@ type Transferable interface {
|
||||
// including calling the API, passing the actual transfer request to transfer
|
||||
// adapters, and dealing with progress, errors and retries
|
||||
type TransferQueue struct {
|
||||
direction transfer.Direction
|
||||
adapter transfer.TransferAdapter
|
||||
adapterInProgress bool
|
||||
adapterResultChan chan transfer.TransferResult
|
||||
@ -55,9 +56,9 @@ type TransferQueue struct {
|
||||
}
|
||||
|
||||
// newTransferQueue builds a TransferQueue, direction and underlying mechanism determined by adapter
|
||||
func newTransferQueue(files int, size int64, dryRun bool, adapter transfer.TransferAdapter) *TransferQueue {
|
||||
func newTransferQueue(files int, size int64, dryRun bool, dir transfer.Direction) *TransferQueue {
|
||||
q := &TransferQueue{
|
||||
adapter: adapter,
|
||||
direction: dir,
|
||||
dryRun: dryRun,
|
||||
meter: progress.NewProgressMeter(files, size, dryRun, config.Config.Getenv("GIT_LFS_PROGRESS")),
|
||||
apic: make(chan Transferable, batchSize),
|
||||
@ -91,6 +92,32 @@ func (q *TransferQueue) Add(t Transferable) {
|
||||
q.apic <- t
|
||||
}
|
||||
|
||||
func (q *TransferQueue) useAdapter(name string) {
|
||||
q.adapterInitMutex.Lock()
|
||||
defer q.adapterInitMutex.Unlock()
|
||||
|
||||
if q.adapter != nil {
|
||||
if q.adapter.Name() == name {
|
||||
// re-use, this is the normal path
|
||||
return
|
||||
}
|
||||
// If the adapter we're using isn't the same as the one we've been
|
||||
// told to use now, must wait for the current one to finish then switch
|
||||
// This will probably never happen but is just in case server starts
|
||||
// changing adapter support in between batches
|
||||
q.finishAdapter()
|
||||
}
|
||||
q.adapter = transfer.NewAdapterOrDefault(name, q.direction)
|
||||
}
|
||||
|
||||
func (q *TransferQueue) finishAdapter() {
|
||||
if q.adapterInProgress {
|
||||
q.adapter.End()
|
||||
q.adapterInProgress = false
|
||||
q.adapter = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (q *TransferQueue) addToAdapter(t Transferable) {
|
||||
|
||||
tr := transfer.NewTransfer(t.Name(), t.Object(), t.Path())
|
||||
@ -110,7 +137,7 @@ func (q *TransferQueue) Skip(size int64) {
|
||||
}
|
||||
|
||||
func (q *TransferQueue) transferKind() string {
|
||||
if q.adapter.Direction() == transfer.Download {
|
||||
if q.direction == transfer.Download {
|
||||
return "download"
|
||||
} else {
|
||||
return "upload"
|
||||
@ -202,10 +229,7 @@ func (q *TransferQueue) Wait() {
|
||||
atomic.StoreUint32(&q.retrying, 0)
|
||||
|
||||
close(q.apic)
|
||||
if q.adapterInProgress {
|
||||
q.adapter.End()
|
||||
q.adapterInProgress = false
|
||||
}
|
||||
q.finishAdapter()
|
||||
close(q.errorc)
|
||||
|
||||
for _, watcher := range q.watchers {
|
||||
@ -250,6 +274,8 @@ func (q *TransferQueue) individualApiRoutine(apiWaiter chan interface{}) {
|
||||
}
|
||||
}
|
||||
|
||||
// Legacy API has no support for anything but basic transfer adapter
|
||||
q.useAdapter(transfer.BasicAdapterName)
|
||||
if obj != nil {
|
||||
t.SetObject(obj)
|
||||
q.meter.Add(t.Name())
|
||||
@ -292,6 +318,8 @@ func (q *TransferQueue) legacyFallback(failedBatch []interface{}) {
|
||||
func (q *TransferQueue) batchApiRoutine() {
|
||||
var startProgress sync.Once
|
||||
|
||||
transferAdapterNames := transfer.GetAdapterNames(q.direction)
|
||||
|
||||
for {
|
||||
batch := q.batcher.Next()
|
||||
if batch == nil {
|
||||
@ -306,7 +334,11 @@ func (q *TransferQueue) batchApiRoutine() {
|
||||
transfers = append(transfers, &api.ObjectResource{Oid: t.Oid(), Size: t.Size()})
|
||||
}
|
||||
|
||||
objects, err := api.Batch(transfers, q.transferKind())
|
||||
if len(transfers) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
objs, adapterName, err := api.Batch(transfers, q.transferKind(), transferAdapterNames)
|
||||
if err != nil {
|
||||
if errutil.IsNotImplementedError(err) {
|
||||
git.Config.SetLocal("", "lfs.batch", "false")
|
||||
@ -327,9 +359,10 @@ func (q *TransferQueue) batchApiRoutine() {
|
||||
continue
|
||||
}
|
||||
|
||||
q.useAdapter(adapterName)
|
||||
startProgress.Do(q.meter.Start)
|
||||
|
||||
for _, o := range objects {
|
||||
for _, o := range objs {
|
||||
if o.Error != nil {
|
||||
q.errorc <- errutil.Errorf(o.Error, "[%v] %v", o.Oid, o.Error.Message)
|
||||
q.Skip(o.Size)
|
||||
|
@ -73,7 +73,7 @@ func NewUploadable(oid, filename string) (*Uploadable, error) {
|
||||
|
||||
// NewUploadQueue builds an UploadQueue, allowing `workers` concurrent uploads.
|
||||
func NewUploadQueue(files int, size int64, dryRun bool) *TransferQueue {
|
||||
return newTransferQueue(files, size, dryRun, transfer.NewUploadAdapter(transfer.BasicAdapterName))
|
||||
return newTransferQueue(files, size, dryRun, transfer.Upload)
|
||||
}
|
||||
|
||||
// ensureFile makes sure that the cleanPath exists before pushing it. If it
|
||||
|
@ -133,8 +133,8 @@ type lfsError struct {
|
||||
func lfsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
repo, err := repoFromLfsUrl(r.URL.Path)
|
||||
if err != nil {
|
||||
w.Write([]byte(err.Error()))
|
||||
w.WriteHeader(500)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
@ -510,15 +510,15 @@ func missingRequiredCreds(w http.ResponseWriter, r *http.Request, repo string) b
|
||||
auth := r.Header.Get("Authorization")
|
||||
user, pass, err := extractAuth(auth)
|
||||
if err != nil {
|
||||
w.Write([]byte(`{"message":"` + err.Error() + `"}`))
|
||||
w.WriteHeader(403)
|
||||
w.Write([]byte(`{"message":"` + err.Error() + `"}`))
|
||||
return true
|
||||
}
|
||||
|
||||
if user != "requirecreds" || pass != "pass" {
|
||||
errmsg := fmt.Sprintf("Got: '%s' => '%s' : '%s'", auth, user, pass)
|
||||
w.Write([]byte(`{"message":"` + errmsg + `"}`))
|
||||
w.WriteHeader(403)
|
||||
w.Write([]byte(`{"message":"` + errmsg + `"}`))
|
||||
return true
|
||||
}
|
||||
|
||||
|
@ -250,7 +250,11 @@ func callBatchApi(op string, objs []TestObject) ([]*api.ObjectResource, error) {
|
||||
for _, o := range objs {
|
||||
apiobjs = append(apiobjs, &api.ObjectResource{Oid: o.Oid, Size: o.Size})
|
||||
}
|
||||
return api.Batch(apiobjs, op)
|
||||
o, _, err := api.Batch(apiobjs, op, []string{"basic"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// Combine 2 slices into one by "randomly" interleaving
|
||||
|
@ -6,6 +6,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/github/git-lfs/api"
|
||||
"github.com/rubyist/tracerx"
|
||||
)
|
||||
|
||||
type Direction int
|
||||
@ -140,6 +141,20 @@ func RegisterNewTransferAdapterFunc(name string, dir Direction, f NewTransferAda
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new adapter by name and direction; default to BasicAdapterName if doesn't exist
|
||||
func NewAdapterOrDefault(name string, dir Direction) TransferAdapter {
|
||||
if len(name) == 0 {
|
||||
name = BasicAdapterName
|
||||
}
|
||||
|
||||
a := NewAdapter(name, dir)
|
||||
if a == nil {
|
||||
tracerx.Printf("Defaulting to basic transfer adapter since %q did not exist", name)
|
||||
a = NewAdapter(BasicAdapterName, dir)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// Create a new adapter by name and direction, or nil if doesn't exist
|
||||
func NewAdapter(name string, dir Direction) TransferAdapter {
|
||||
funcMutex.Lock()
|
||||
@ -158,12 +173,12 @@ func NewAdapter(name string, dir Direction) TransferAdapter {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create a new download adapter by name, or nil if doesn't exist
|
||||
// Create a new download adapter by name, or BasicAdapterName if doesn't exist
|
||||
func NewDownloadAdapter(name string) TransferAdapter {
|
||||
return NewAdapter(name, Download)
|
||||
return NewAdapterOrDefault(name, Download)
|
||||
}
|
||||
|
||||
// Create a new upload adapter by name, or nil if doesn't exist
|
||||
// Create a new upload adapter by name, or BasicAdapterName if doesn't exist
|
||||
func NewUploadAdapter(name string) TransferAdapter {
|
||||
return NewAdapter(name, Upload)
|
||||
return NewAdapterOrDefault(name, Upload)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user