2017-01-04 16:33:40 +00:00
|
|
|
package tq
|
|
|
|
|
|
|
|
import (
|
2017-04-05 22:43:00 +00:00
|
|
|
"time"
|
|
|
|
|
2021-09-01 19:41:10 +00:00
|
|
|
"github.com/git-lfs/git-lfs/v3/errors"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/git"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/lfsapi"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/lfshttp"
|
2021-12-14 17:28:16 +00:00
|
|
|
"github.com/git-lfs/git-lfs/v3/tr"
|
2017-01-04 17:11:16 +00:00
|
|
|
"github.com/rubyist/tracerx"
|
2017-01-04 16:33:40 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type tqClient struct {
|
2021-02-10 19:52:37 +00:00
|
|
|
maxRetries int
|
2017-01-04 16:33:40 +00:00
|
|
|
*lfsapi.Client
|
|
|
|
}
|
|
|
|
|
2018-01-05 18:12:57 +00:00
|
|
|
type batchRef struct {
|
|
|
|
Name string `json:"name,omitempty"`
|
|
|
|
}
|
|
|
|
|
2017-01-04 16:33:40 +00:00
|
|
|
type batchRequest struct {
|
2017-01-06 22:51:49 +00:00
|
|
|
Operation string `json:"operation"`
|
|
|
|
Objects []*Transfer `json:"objects"`
|
|
|
|
TransferAdapterNames []string `json:"transfers,omitempty"`
|
2018-01-05 18:12:57 +00:00
|
|
|
Ref *batchRef `json:"ref"`
|
2021-06-02 20:13:49 +00:00
|
|
|
HashAlgorithm string `json:"hash_algo"`
|
2017-01-04 16:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-01-09 20:00:59 +00:00
|
|
|
type BatchResponse struct {
|
2017-01-06 22:51:49 +00:00
|
|
|
Objects []*Transfer `json:"objects"`
|
2017-01-09 20:04:58 +00:00
|
|
|
TransferAdapterName string `json:"transfer"`
|
2021-06-02 20:13:49 +00:00
|
|
|
HashAlgorithm string `json:"hash_algo"`
|
2018-09-06 21:42:41 +00:00
|
|
|
endpoint lfshttp.Endpoint
|
2017-01-04 16:33:40 +00:00
|
|
|
}
|
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func Batch(m Manifest, dir Direction, remote string, remoteRef *git.Ref, objects []*Transfer) (*BatchResponse, error) {
|
2017-01-07 03:13:31 +00:00
|
|
|
if len(objects) == 0 {
|
2017-01-09 20:50:19 +00:00
|
|
|
return &BatchResponse{}, nil
|
2017-01-07 03:13:31 +00:00
|
|
|
}
|
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
cm := m.Upgrade()
|
|
|
|
|
|
|
|
return cm.batchClient().Batch(remote, &batchRequest{
|
2017-01-09 19:52:36 +00:00
|
|
|
Operation: dir.String(),
|
2017-01-07 03:13:31 +00:00
|
|
|
Objects: objects,
|
|
|
|
TransferAdapterNames: m.GetAdapterNames(dir),
|
2018-01-05 18:12:57 +00:00
|
|
|
Ref: &batchRef{Name: remoteRef.Refspec()},
|
2021-06-02 20:13:49 +00:00
|
|
|
HashAlgorithm: "sha256",
|
2017-01-09 20:50:19 +00:00
|
|
|
})
|
2017-01-07 03:13:31 +00:00
|
|
|
}
|
|
|
|
|
2021-02-10 19:52:37 +00:00
|
|
|
type BatchClient interface {
|
|
|
|
Batch(remote string, bReq *batchRequest) (*BatchResponse, error)
|
|
|
|
MaxRetries() int
|
|
|
|
SetMaxRetries(n int)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *tqClient) MaxRetries() int {
|
|
|
|
return c.maxRetries
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *tqClient) SetMaxRetries(n int) {
|
|
|
|
c.maxRetries = n
|
|
|
|
}
|
|
|
|
|
2017-01-09 20:50:19 +00:00
|
|
|
func (c *tqClient) Batch(remote string, bReq *batchRequest) (*BatchResponse, error) {
|
2017-01-09 20:00:59 +00:00
|
|
|
bRes := &BatchResponse{}
|
2017-01-04 16:33:40 +00:00
|
|
|
if len(bReq.Objects) == 0 {
|
2017-01-09 20:50:19 +00:00
|
|
|
return bRes, nil
|
2017-01-04 16:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if len(bReq.TransferAdapterNames) == 1 && bReq.TransferAdapterNames[0] == "basic" {
|
|
|
|
bReq.TransferAdapterNames = nil
|
|
|
|
}
|
|
|
|
|
Don't fail if we lack objects the server has
A Git LFS client may not have the entire history of the objects for the
repository. However, in some situations, we traverse the entire history
of a branch when pushing it, meaning that we need to process every
LFS object in the history of that branch. If the objects for the entire
history are not present, we currently fail to push.
Instead, let's mark objects we don't have on disk as missing and only
fail when we would need to upload those objects. We'll know the server
has the objects if the batch response provides no actions to take for
them when we request an upload. Pass the missing flag down through the
code, and always set it to false for non-uploads.
If for some reason we fail to properly flag a missing object, we will
still fail later on when we cannot open the file, just in a messier and
more poorly controlled way. The technique used here will attempt to
abort the batch as soon as we notice a problem, which means that in the
common case (less than 100 objects) we won't have transferred any
objects, so the user can notice the failure as soon as possible.
Update the tests to look for a string which will occur in the error
message, since we no longer produce the system error message for ENOENT.
2019-04-30 19:18:18 +00:00
|
|
|
missing := make(map[string]bool)
|
|
|
|
for _, obj := range bReq.Objects {
|
|
|
|
missing[obj.Oid] = obj.Missing
|
|
|
|
}
|
|
|
|
|
2017-01-09 20:00:59 +00:00
|
|
|
bRes.endpoint = c.Endpoints.Endpoint(bReq.Operation, remote)
|
2017-04-05 22:43:00 +00:00
|
|
|
requestedAt := time.Now()
|
|
|
|
|
2017-08-24 21:51:49 +00:00
|
|
|
req, err := c.NewRequest("POST", bRes.endpoint, "objects/batch", bReq)
|
|
|
|
if err != nil {
|
2021-12-14 17:28:16 +00:00
|
|
|
return nil, errors.Wrap(err, tr.Tr.Get("batch request"))
|
2017-08-24 21:51:49 +00:00
|
|
|
}
|
2017-08-18 00:52:57 +00:00
|
|
|
|
2017-08-24 21:51:49 +00:00
|
|
|
tracerx.Printf("api: batch %d files", len(bReq.Objects))
|
2017-01-04 17:11:16 +00:00
|
|
|
|
2018-09-06 21:42:41 +00:00
|
|
|
req = c.Client.LogRequest(req, "lfs.batch")
|
2021-02-10 19:52:37 +00:00
|
|
|
res, err := c.DoAPIRequestWithAuth(remote, lfshttp.WithRetries(req, c.MaxRetries()))
|
2017-08-24 21:51:49 +00:00
|
|
|
if err != nil {
|
2017-01-04 17:11:16 +00:00
|
|
|
tracerx.Printf("api error: %s", err)
|
2021-12-14 17:28:16 +00:00
|
|
|
return nil, errors.Wrap(err, tr.Tr.Get("batch response"))
|
2017-01-04 17:11:16 +00:00
|
|
|
}
|
|
|
|
|
2018-09-06 21:42:41 +00:00
|
|
|
if err := lfshttp.DecodeJSON(res, bRes); err != nil {
|
2021-12-14 17:28:16 +00:00
|
|
|
return bRes, errors.Wrap(err, tr.Tr.Get("batch response"))
|
2017-01-04 17:20:40 +00:00
|
|
|
}
|
|
|
|
|
2021-06-02 20:13:49 +00:00
|
|
|
if bRes.HashAlgorithm != "" && bRes.HashAlgorithm != "sha256" {
|
2021-12-14 17:28:16 +00:00
|
|
|
return bRes, errors.Wrap(errors.New(tr.Tr.Get("unsupported hash algorithm")), tr.Tr.Get("batch response"))
|
2021-06-02 20:13:49 +00:00
|
|
|
}
|
|
|
|
|
2017-01-04 17:11:16 +00:00
|
|
|
if res.StatusCode != 200 {
|
2018-09-06 21:42:41 +00:00
|
|
|
return nil, lfshttp.NewStatusCodeError(res)
|
2017-01-04 16:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-04-05 22:43:00 +00:00
|
|
|
for _, obj := range bRes.Objects {
|
Don't fail if we lack objects the server has
A Git LFS client may not have the entire history of the objects for the
repository. However, in some situations, we traverse the entire history
of a branch when pushing it, meaning that we need to process every
LFS object in the history of that branch. If the objects for the entire
history are not present, we currently fail to push.
Instead, let's mark objects we don't have on disk as missing and only
fail when we would need to upload those objects. We'll know the server
has the objects if the batch response provides no actions to take for
them when we request an upload. Pass the missing flag down through the
code, and always set it to false for non-uploads.
If for some reason we fail to properly flag a missing object, we will
still fail later on when we cannot open the file, just in a messier and
more poorly controlled way. The technique used here will attempt to
abort the batch as soon as we notice a problem, which means that in the
common case (less than 100 objects) we won't have transferred any
objects, so the user can notice the failure as soon as possible.
Update the tests to look for a string which will occur in the error
message, since we no longer produce the system error message for ENOENT.
2019-04-30 19:18:18 +00:00
|
|
|
obj.Missing = missing[obj.Oid]
|
2017-04-05 22:43:00 +00:00
|
|
|
for _, a := range obj.Actions {
|
|
|
|
a.createdAt = requestedAt
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-09 20:50:19 +00:00
|
|
|
return bRes, nil
|
2017-01-04 16:33:40 +00:00
|
|
|
}
|