2016-12-12 00:28:47 +00:00
|
|
|
package tq
|
2016-08-09 21:43:29 +00:00
|
|
|
|
|
|
|
import (
|
2019-07-31 13:46:39 +00:00
|
|
|
"strings"
|
2016-08-09 21:43:29 +00:00
|
|
|
"sync"
|
|
|
|
|
2021-09-01 19:41:10 +00:00
|
|
|
"github.com/git-lfs/git-lfs/v3/config"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/fs"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/lfsapi"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/ssh"
|
2016-08-09 21:43:29 +00:00
|
|
|
"github.com/rubyist/tracerx"
|
|
|
|
)
|
|
|
|
|
2016-12-13 21:14:30 +00:00
|
|
|
const (
|
2017-02-18 20:22:49 +00:00
|
|
|
defaultMaxRetries = 8
|
2020-04-15 03:13:27 +00:00
|
|
|
defaultMaxRetryDelay = 10
|
2017-08-12 04:23:07 +00:00
|
|
|
defaultConcurrentTransfers = 8
|
2016-12-13 21:14:30 +00:00
|
|
|
)
|
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
type Manifest interface {
|
|
|
|
APIClient() *lfsapi.Client
|
|
|
|
MaxRetries() int
|
|
|
|
MaxRetryDelay() int
|
|
|
|
ConcurrentTransfers() int
|
|
|
|
IsStandaloneTransfer() bool
|
|
|
|
batchClient() BatchClient
|
|
|
|
GetAdapterNames(dir Direction) []string
|
|
|
|
GetDownloadAdapterNames() []string
|
|
|
|
GetUploadAdapterNames() []string
|
|
|
|
getAdapterNames(adapters map[string]NewAdapterFunc) []string
|
|
|
|
RegisterNewAdapterFunc(name string, dir Direction, f NewAdapterFunc)
|
|
|
|
NewAdapterOrDefault(name string, dir Direction) Adapter
|
|
|
|
NewAdapter(name string, dir Direction) Adapter
|
|
|
|
NewDownloadAdapter(name string) Adapter
|
|
|
|
NewUploadAdapter(name string) Adapter
|
|
|
|
Upgrade() *concreteManifest
|
|
|
|
Upgraded() bool
|
|
|
|
}
|
|
|
|
|
tq: avoid spawning SSH process needlessly
When a user invokes `git archive` with LFS files, `git lfs
filter-process` is invoked to smudge the LFS files. However, currently
when we instantiate the manifest object as part of that, an attempt is
made to connect to the remote using SSH, which we don't want to do
unless necessary. For example, if the user already has all the files
locally, the network connection is needless and serves only to waste
resources.
In the previous commit, we made our manifest an abstract interface with
a single implementing type: a concrete manifest. Now, introduce a lazy
manifest, which can upgrade to a concrete manifest but doesn't
instantiate one until that happens. This allows us to instantiate a
manifest without making the SSH connection, and we can delay the SSH
connection until it's really needed, if at all.
Add a test for this case as well.
2023-03-10 16:18:00 +00:00
|
|
|
type lazyManifest struct {
|
|
|
|
f *fs.Filesystem
|
|
|
|
apiClient *lfsapi.Client
|
|
|
|
operation string
|
|
|
|
remote string
|
|
|
|
m *concreteManifest
|
|
|
|
}
|
|
|
|
|
|
|
|
func newLazyManifest(f *fs.Filesystem, apiClient *lfsapi.Client, operation, remote string) *lazyManifest {
|
|
|
|
return &lazyManifest{
|
|
|
|
f: f,
|
|
|
|
apiClient: apiClient,
|
|
|
|
operation: operation,
|
|
|
|
remote: remote,
|
|
|
|
m: nil,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) APIClient() *lfsapi.Client {
|
|
|
|
return m.Upgrade().APIClient()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) MaxRetries() int {
|
|
|
|
return m.Upgrade().MaxRetries()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) MaxRetryDelay() int {
|
|
|
|
return m.Upgrade().MaxRetryDelay()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) ConcurrentTransfers() int {
|
|
|
|
return m.Upgrade().ConcurrentTransfers()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) IsStandaloneTransfer() bool {
|
|
|
|
return m.Upgrade().IsStandaloneTransfer()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) batchClient() BatchClient {
|
|
|
|
return m.Upgrade().batchClient()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) GetAdapterNames(dir Direction) []string {
|
|
|
|
return m.Upgrade().GetAdapterNames(dir)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) GetDownloadAdapterNames() []string {
|
|
|
|
return m.Upgrade().GetDownloadAdapterNames()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) GetUploadAdapterNames() []string {
|
|
|
|
return m.Upgrade().GetUploadAdapterNames()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) getAdapterNames(adapters map[string]NewAdapterFunc) []string {
|
|
|
|
return m.Upgrade().getAdapterNames(adapters)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) RegisterNewAdapterFunc(name string, dir Direction, f NewAdapterFunc) {
|
|
|
|
m.Upgrade().RegisterNewAdapterFunc(name, dir, f)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) NewAdapterOrDefault(name string, dir Direction) Adapter {
|
|
|
|
return m.Upgrade().NewAdapterOrDefault(name, dir)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) NewAdapter(name string, dir Direction) Adapter {
|
|
|
|
return m.Upgrade().NewAdapter(name, dir)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) NewDownloadAdapter(name string) Adapter {
|
|
|
|
return m.Upgrade().NewDownloadAdapter(name)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) NewUploadAdapter(name string) Adapter {
|
|
|
|
return m.Upgrade().NewUploadAdapter(name)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) Upgrade() *concreteManifest {
|
|
|
|
if m.m == nil {
|
|
|
|
m.m = newConcreteManifest(m.f, m.apiClient, m.operation, m.remote)
|
|
|
|
}
|
|
|
|
return m.m
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *lazyManifest) Upgraded() bool {
|
|
|
|
return m.m != nil
|
|
|
|
}
|
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
type concreteManifest struct {
|
2017-07-20 14:09:14 +00:00
|
|
|
// maxRetries is the maximum number of retries a single object can
|
2020-04-15 03:13:27 +00:00
|
|
|
// attempt to make before it will be dropped. maxRetryDelay is the maximum
|
|
|
|
// time in seconds to wait between retry attempts when using backoff.
|
2017-07-20 14:09:14 +00:00
|
|
|
maxRetries int
|
2020-04-15 03:13:27 +00:00
|
|
|
maxRetryDelay int
|
2017-07-20 14:09:14 +00:00
|
|
|
concurrentTransfers int
|
|
|
|
basicTransfersOnly bool
|
|
|
|
standaloneTransferAgent string
|
|
|
|
tusTransfersAllowed bool
|
|
|
|
downloadAdapterFuncs map[string]NewAdapterFunc
|
|
|
|
uploadAdapterFuncs map[string]NewAdapterFunc
|
2017-10-25 17:46:37 +00:00
|
|
|
fs *fs.Filesystem
|
2017-07-20 14:09:14 +00:00
|
|
|
apiClient *lfsapi.Client
|
2021-03-17 18:19:20 +00:00
|
|
|
sshTransfer *ssh.SSHTransfer
|
2021-02-10 19:52:37 +00:00
|
|
|
batchClientAdapter BatchClient
|
2017-07-20 14:09:14 +00:00
|
|
|
mu sync.Mutex
|
2016-08-09 21:43:29 +00:00
|
|
|
}
|
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) APIClient() *lfsapi.Client {
|
2017-01-04 16:33:40 +00:00
|
|
|
return m.apiClient
|
|
|
|
}
|
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) MaxRetries() int {
|
2016-12-13 23:48:54 +00:00
|
|
|
return m.maxRetries
|
|
|
|
}
|
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) MaxRetryDelay() int {
|
2020-04-15 03:13:27 +00:00
|
|
|
return m.maxRetryDelay
|
|
|
|
}
|
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) ConcurrentTransfers() int {
|
2016-12-13 23:48:54 +00:00
|
|
|
return m.concurrentTransfers
|
|
|
|
}
|
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) IsStandaloneTransfer() bool {
|
2017-09-20 12:30:03 +00:00
|
|
|
return m.standaloneTransferAgent != ""
|
|
|
|
}
|
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) batchClient() BatchClient {
|
2017-09-11 14:44:31 +00:00
|
|
|
if r := m.MaxRetries(); r > 0 {
|
2021-02-10 19:52:37 +00:00
|
|
|
m.batchClientAdapter.SetMaxRetries(r)
|
2017-09-11 14:44:31 +00:00
|
|
|
}
|
2021-02-10 19:52:37 +00:00
|
|
|
return m.batchClientAdapter
|
2017-01-09 20:14:10 +00:00
|
|
|
}
|
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) Upgrade() *concreteManifest {
|
|
|
|
return m
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *concreteManifest) Upgraded() bool {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewManifest(f *fs.Filesystem, apiClient *lfsapi.Client, operation, remote string) Manifest {
|
tq: avoid spawning SSH process needlessly
When a user invokes `git archive` with LFS files, `git lfs
filter-process` is invoked to smudge the LFS files. However, currently
when we instantiate the manifest object as part of that, an attempt is
made to connect to the remote using SSH, which we don't want to do
unless necessary. For example, if the user already has all the files
locally, the network connection is needless and serves only to waste
resources.
In the previous commit, we made our manifest an abstract interface with
a single implementing type: a concrete manifest. Now, introduce a lazy
manifest, which can upgrade to a concrete manifest but doesn't
instantiate one until that happens. This allows us to instantiate a
manifest without making the SSH connection, and we can delay the SSH
connection until it's really needed, if at all.
Add a test for this case as well.
2023-03-10 16:18:00 +00:00
|
|
|
return newLazyManifest(f, apiClient, operation, remote)
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func newConcreteManifest(f *fs.Filesystem, apiClient *lfsapi.Client, operation, remote string) *concreteManifest {
|
2017-10-25 17:46:37 +00:00
|
|
|
if apiClient == nil {
|
2017-10-25 21:33:20 +00:00
|
|
|
cli, err := lfsapi.NewClient(nil)
|
2017-10-25 17:46:37 +00:00
|
|
|
if err != nil {
|
|
|
|
tracerx.Printf("unable to init tq.Manifest: %s", err)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
apiClient = cli
|
2017-01-03 22:48:30 +00:00
|
|
|
}
|
|
|
|
|
2021-05-07 16:56:15 +00:00
|
|
|
sshTransfer := apiClient.SSHTransfer(operation, remote)
|
2022-10-05 14:03:18 +00:00
|
|
|
useSSHMultiplexing := false
|
|
|
|
if sshTransfer != nil {
|
|
|
|
useSSHMultiplexing = sshTransfer.IsMultiplexingEnabled()
|
|
|
|
}
|
2021-03-17 18:19:20 +00:00
|
|
|
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
m := &concreteManifest{
|
2017-10-25 17:46:37 +00:00
|
|
|
fs: f,
|
2017-01-04 16:33:40 +00:00
|
|
|
apiClient: apiClient,
|
2021-02-10 19:52:37 +00:00
|
|
|
batchClientAdapter: &tqClient{Client: apiClient},
|
2016-12-12 00:55:14 +00:00
|
|
|
downloadAdapterFuncs: make(map[string]NewAdapterFunc),
|
|
|
|
uploadAdapterFuncs: make(map[string]NewAdapterFunc),
|
2021-03-17 18:19:20 +00:00
|
|
|
sshTransfer: sshTransfer,
|
2016-08-09 21:43:29 +00:00
|
|
|
}
|
|
|
|
|
2016-12-13 22:38:10 +00:00
|
|
|
var tusAllowed bool
|
2017-01-04 16:05:29 +00:00
|
|
|
if git := apiClient.GitEnv(); git != nil {
|
2016-12-13 22:38:10 +00:00
|
|
|
if v := git.Int("lfs.transfer.maxretries", 0); v > 0 {
|
2016-12-13 23:48:54 +00:00
|
|
|
m.maxRetries = v
|
2016-12-13 22:38:10 +00:00
|
|
|
}
|
2020-04-15 03:13:27 +00:00
|
|
|
if v := git.Int("lfs.transfer.maxretrydelay", -1); v > -1 {
|
|
|
|
m.maxRetryDelay = v
|
|
|
|
}
|
2016-12-13 22:38:10 +00:00
|
|
|
if v := git.Int("lfs.concurrenttransfers", 0); v > 0 {
|
2016-12-13 23:48:54 +00:00
|
|
|
m.concurrentTransfers = v
|
2016-12-13 22:38:10 +00:00
|
|
|
}
|
|
|
|
m.basicTransfersOnly = git.Bool("lfs.basictransfersonly", false)
|
2017-09-17 16:41:30 +00:00
|
|
|
m.standaloneTransferAgent = findStandaloneTransfer(
|
|
|
|
apiClient, operation, remote,
|
|
|
|
)
|
2016-12-13 22:38:10 +00:00
|
|
|
tusAllowed = git.Bool("lfs.tustransfers", false)
|
|
|
|
configureCustomAdapters(git, m)
|
|
|
|
}
|
|
|
|
|
2016-12-13 23:48:54 +00:00
|
|
|
if m.maxRetries < 1 {
|
|
|
|
m.maxRetries = defaultMaxRetries
|
2016-12-13 20:32:25 +00:00
|
|
|
}
|
2020-04-15 03:13:27 +00:00
|
|
|
if m.maxRetryDelay < 1 {
|
|
|
|
m.maxRetryDelay = defaultMaxRetryDelay
|
|
|
|
}
|
2016-12-13 23:48:54 +00:00
|
|
|
|
2017-01-04 21:46:30 +00:00
|
|
|
if m.concurrentTransfers < 1 {
|
2016-12-13 23:48:54 +00:00
|
|
|
m.concurrentTransfers = defaultConcurrentTransfers
|
2016-12-13 20:06:58 +00:00
|
|
|
}
|
|
|
|
|
2021-03-17 18:19:20 +00:00
|
|
|
if sshTransfer != nil {
|
2022-10-05 14:03:18 +00:00
|
|
|
if !useSSHMultiplexing {
|
|
|
|
m.concurrentTransfers = 1
|
|
|
|
}
|
|
|
|
|
2021-03-17 18:19:20 +00:00
|
|
|
// Multiple concurrent transfers are not yet supported.
|
|
|
|
m.batchClientAdapter = &SSHBatchClient{
|
|
|
|
maxRetries: m.maxRetries,
|
|
|
|
transfer: sshTransfer,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-09 21:43:29 +00:00
|
|
|
configureBasicDownloadAdapter(m)
|
|
|
|
configureBasicUploadAdapter(m)
|
2016-12-13 22:38:10 +00:00
|
|
|
if tusAllowed {
|
2016-08-09 21:43:29 +00:00
|
|
|
configureTusAdapter(m)
|
|
|
|
}
|
2021-03-17 18:19:20 +00:00
|
|
|
configureSSHAdapter(m)
|
2016-12-14 16:53:02 +00:00
|
|
|
return m
|
2016-12-13 21:14:30 +00:00
|
|
|
}
|
|
|
|
|
2019-07-31 13:46:39 +00:00
|
|
|
func findDefaultStandaloneTransfer(url string) string {
|
|
|
|
if strings.HasPrefix(url, "file://") {
|
|
|
|
return standaloneFileName
|
|
|
|
}
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
2017-09-17 16:41:30 +00:00
|
|
|
func findStandaloneTransfer(client *lfsapi.Client, operation, remote string) string {
|
|
|
|
if operation == "" || remote == "" {
|
|
|
|
v, _ := client.GitEnv().Get("lfs.standalonetransferagent")
|
|
|
|
return v
|
|
|
|
}
|
|
|
|
|
2021-03-17 18:19:20 +00:00
|
|
|
ep := client.Endpoints.Endpoint(operation, remote)
|
2017-09-17 16:41:30 +00:00
|
|
|
uc := config.NewURLConfig(client.GitEnv())
|
|
|
|
v, ok := uc.Get("lfs", ep.Url, "standalonetransferagent")
|
|
|
|
if !ok {
|
2023-10-17 17:21:35 +00:00
|
|
|
return findDefaultStandaloneTransfer(ep.Url)
|
2017-09-17 16:41:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return v
|
|
|
|
}
|
|
|
|
|
2016-08-09 21:43:29 +00:00
|
|
|
// GetAdapterNames returns a list of the names of adapters available to be created
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) GetAdapterNames(dir Direction) []string {
|
2016-08-09 21:43:29 +00:00
|
|
|
switch dir {
|
|
|
|
case Upload:
|
|
|
|
return m.GetUploadAdapterNames()
|
|
|
|
case Download:
|
|
|
|
return m.GetDownloadAdapterNames()
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetDownloadAdapterNames returns a list of the names of download adapters available to be created
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) GetDownloadAdapterNames() []string {
|
2016-08-09 21:43:29 +00:00
|
|
|
return m.getAdapterNames(m.downloadAdapterFuncs)
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetUploadAdapterNames returns a list of the names of upload adapters available to be created
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) GetUploadAdapterNames() []string {
|
2016-08-09 21:43:29 +00:00
|
|
|
return m.getAdapterNames(m.uploadAdapterFuncs)
|
|
|
|
}
|
|
|
|
|
2016-08-10 14:39:57 +00:00
|
|
|
// getAdapterNames returns a list of the names of adapters available to be created
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) getAdapterNames(adapters map[string]NewAdapterFunc) []string {
|
2016-12-13 22:38:10 +00:00
|
|
|
if m.basicTransfersOnly {
|
2016-08-09 21:43:29 +00:00
|
|
|
return []string{BasicAdapterName}
|
|
|
|
}
|
|
|
|
|
|
|
|
m.mu.Lock()
|
|
|
|
defer m.mu.Unlock()
|
|
|
|
|
|
|
|
ret := make([]string, 0, len(adapters))
|
|
|
|
for n, _ := range adapters {
|
|
|
|
ret = append(ret, n)
|
|
|
|
}
|
|
|
|
return ret
|
|
|
|
}
|
|
|
|
|
|
|
|
// RegisterNewTransferAdapterFunc registers a new function for creating upload
|
|
|
|
// or download adapters. If a function with that name & direction is already
|
|
|
|
// registered, it is overridden
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) RegisterNewAdapterFunc(name string, dir Direction, f NewAdapterFunc) {
|
2016-08-09 21:43:29 +00:00
|
|
|
m.mu.Lock()
|
|
|
|
defer m.mu.Unlock()
|
|
|
|
|
|
|
|
switch dir {
|
|
|
|
case Upload:
|
|
|
|
m.uploadAdapterFuncs[name] = f
|
|
|
|
case Download:
|
|
|
|
m.downloadAdapterFuncs[name] = f
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a new adapter by name and direction; default to BasicAdapterName if doesn't exist
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) NewAdapterOrDefault(name string, dir Direction) Adapter {
|
2016-08-09 21:43:29 +00:00
|
|
|
if len(name) == 0 {
|
|
|
|
name = BasicAdapterName
|
|
|
|
}
|
|
|
|
|
|
|
|
a := m.NewAdapter(name, dir)
|
|
|
|
if a == nil {
|
|
|
|
tracerx.Printf("Defaulting to basic transfer adapter since %q did not exist", name)
|
|
|
|
a = m.NewAdapter(BasicAdapterName, dir)
|
|
|
|
}
|
|
|
|
return a
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a new adapter by name and direction, or nil if doesn't exist
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) NewAdapter(name string, dir Direction) Adapter {
|
2016-08-09 21:43:29 +00:00
|
|
|
m.mu.Lock()
|
|
|
|
defer m.mu.Unlock()
|
|
|
|
|
|
|
|
switch dir {
|
|
|
|
case Upload:
|
|
|
|
if u, ok := m.uploadAdapterFuncs[name]; ok {
|
|
|
|
return u(name, dir)
|
|
|
|
}
|
|
|
|
case Download:
|
|
|
|
if d, ok := m.downloadAdapterFuncs[name]; ok {
|
|
|
|
return d(name, dir)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a new download adapter by name, or BasicAdapterName if doesn't exist
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) NewDownloadAdapter(name string) Adapter {
|
2016-08-09 21:43:29 +00:00
|
|
|
return m.NewAdapterOrDefault(name, Download)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a new upload adapter by name, or BasicAdapterName if doesn't exist
|
tq: make Manifest an interface
Right now, any time we instantiate a Manifest object, we create an API
client, and when we create the API client, if we're using SSH, we try to
make a connection to the server. However, we often instantiate a
Manifest object when performing various functionality such as smudging
data, which means that when a user creates an archive locally, they can
be prompted for an SSH password, which is undesirable.
Let's take a first step to fixing this by making Manifest an interface.
Right now, it has one concrete version, a concreteManifest, which can be
used to access the internals, and we provide methods to upgrade it from
the interface to the concrete type and determine whether it's upgraded
or not. We attempt to upgrade it any time we need to access its
internals. In the future, we'll also offer a lazyManifest, which is
lazy and will only instantiate the concreteManifest inside when we
attempt to upgrade it to the latter. But for now, only implement the
concreteManifest to make it clearer what's changing.
Similarly, we make our TransferQueue upgradable so that we don't
upgrade its Manifest right away.
In both cases, we'll want to use the lazyManifest to delay the
instantiation of the API client (and hence the starting of the SSH
connection) in a future commit.
2023-03-10 15:24:21 +00:00
|
|
|
func (m *concreteManifest) NewUploadAdapter(name string) Adapter {
|
2016-08-09 21:43:29 +00:00
|
|
|
return m.NewAdapterOrDefault(name, Upload)
|
|
|
|
}
|
2016-12-13 19:59:16 +00:00
|
|
|
|
2016-12-13 22:40:47 +00:00
|
|
|
// Env is any object with a config.Environment interface.
|
|
|
|
type Env interface {
|
2016-12-13 19:59:16 +00:00
|
|
|
Get(key string) (val string, ok bool)
|
2017-04-12 21:29:11 +00:00
|
|
|
GetAll(key string) []string
|
2016-12-13 19:59:16 +00:00
|
|
|
Bool(key string, def bool) (val bool)
|
|
|
|
Int(key string, def int) (val int)
|
2017-04-12 21:29:11 +00:00
|
|
|
All() map[string][]string
|
2016-12-13 19:59:16 +00:00
|
|
|
}
|