2016-12-12 00:28:47 +00:00
|
|
|
package tq
|
2016-08-09 21:43:29 +00:00
|
|
|
|
|
|
|
import (
|
2019-07-31 13:46:39 +00:00
|
|
|
"strings"
|
2016-08-09 21:43:29 +00:00
|
|
|
"sync"
|
|
|
|
|
2017-09-17 16:41:30 +00:00
|
|
|
"github.com/git-lfs/git-lfs/config"
|
2017-10-25 17:46:37 +00:00
|
|
|
"github.com/git-lfs/git-lfs/fs"
|
2017-01-03 22:48:30 +00:00
|
|
|
"github.com/git-lfs/git-lfs/lfsapi"
|
2021-03-17 18:19:20 +00:00
|
|
|
"github.com/git-lfs/git-lfs/ssh"
|
2016-08-09 21:43:29 +00:00
|
|
|
"github.com/rubyist/tracerx"
|
|
|
|
)
|
|
|
|
|
2016-12-13 21:14:30 +00:00
|
|
|
const (
|
2017-02-18 20:22:49 +00:00
|
|
|
defaultMaxRetries = 8
|
2020-04-15 03:13:27 +00:00
|
|
|
defaultMaxRetryDelay = 10
|
2017-08-12 04:23:07 +00:00
|
|
|
defaultConcurrentTransfers = 8
|
2016-12-13 21:14:30 +00:00
|
|
|
)
|
|
|
|
|
2016-08-09 21:43:29 +00:00
|
|
|
type Manifest struct {
|
2017-07-20 14:09:14 +00:00
|
|
|
// maxRetries is the maximum number of retries a single object can
|
2020-04-15 03:13:27 +00:00
|
|
|
// attempt to make before it will be dropped. maxRetryDelay is the maximum
|
|
|
|
// time in seconds to wait between retry attempts when using backoff.
|
2017-07-20 14:09:14 +00:00
|
|
|
maxRetries int
|
2020-04-15 03:13:27 +00:00
|
|
|
maxRetryDelay int
|
2017-07-20 14:09:14 +00:00
|
|
|
concurrentTransfers int
|
|
|
|
basicTransfersOnly bool
|
|
|
|
standaloneTransferAgent string
|
|
|
|
tusTransfersAllowed bool
|
|
|
|
downloadAdapterFuncs map[string]NewAdapterFunc
|
|
|
|
uploadAdapterFuncs map[string]NewAdapterFunc
|
2017-10-25 17:46:37 +00:00
|
|
|
fs *fs.Filesystem
|
2017-07-20 14:09:14 +00:00
|
|
|
apiClient *lfsapi.Client
|
2021-03-17 18:19:20 +00:00
|
|
|
sshTransfer *ssh.SSHTransfer
|
2021-02-10 19:52:37 +00:00
|
|
|
batchClientAdapter BatchClient
|
2017-07-20 14:09:14 +00:00
|
|
|
mu sync.Mutex
|
2016-08-09 21:43:29 +00:00
|
|
|
}
|
|
|
|
|
2017-01-04 16:33:40 +00:00
|
|
|
func (m *Manifest) APIClient() *lfsapi.Client {
|
|
|
|
return m.apiClient
|
|
|
|
}
|
|
|
|
|
2016-12-13 23:48:54 +00:00
|
|
|
func (m *Manifest) MaxRetries() int {
|
|
|
|
return m.maxRetries
|
|
|
|
}
|
|
|
|
|
2020-04-15 03:13:27 +00:00
|
|
|
func (m *Manifest) MaxRetryDelay() int {
|
|
|
|
return m.maxRetryDelay
|
|
|
|
}
|
|
|
|
|
2016-12-13 23:48:54 +00:00
|
|
|
func (m *Manifest) ConcurrentTransfers() int {
|
|
|
|
return m.concurrentTransfers
|
|
|
|
}
|
|
|
|
|
2017-09-20 12:30:03 +00:00
|
|
|
func (m *Manifest) IsStandaloneTransfer() bool {
|
|
|
|
return m.standaloneTransferAgent != ""
|
|
|
|
}
|
|
|
|
|
2021-02-10 19:52:37 +00:00
|
|
|
func (m *Manifest) batchClient() BatchClient {
|
2017-09-11 14:44:31 +00:00
|
|
|
if r := m.MaxRetries(); r > 0 {
|
2021-02-10 19:52:37 +00:00
|
|
|
m.batchClientAdapter.SetMaxRetries(r)
|
2017-09-11 14:44:31 +00:00
|
|
|
}
|
2021-02-10 19:52:37 +00:00
|
|
|
return m.batchClientAdapter
|
2017-01-09 20:14:10 +00:00
|
|
|
}
|
|
|
|
|
2017-10-25 17:46:37 +00:00
|
|
|
func NewManifest(f *fs.Filesystem, apiClient *lfsapi.Client, operation, remote string) *Manifest {
|
|
|
|
if apiClient == nil {
|
2017-10-25 21:33:20 +00:00
|
|
|
cli, err := lfsapi.NewClient(nil)
|
2017-10-25 17:46:37 +00:00
|
|
|
if err != nil {
|
|
|
|
tracerx.Printf("unable to init tq.Manifest: %s", err)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
apiClient = cli
|
2017-01-03 22:48:30 +00:00
|
|
|
}
|
|
|
|
|
2021-05-07 16:56:15 +00:00
|
|
|
sshTransfer := apiClient.SSHTransfer(operation, remote)
|
2021-03-17 18:19:20 +00:00
|
|
|
|
2016-12-13 22:38:10 +00:00
|
|
|
m := &Manifest{
|
2017-10-25 17:46:37 +00:00
|
|
|
fs: f,
|
2017-01-04 16:33:40 +00:00
|
|
|
apiClient: apiClient,
|
2021-02-10 19:52:37 +00:00
|
|
|
batchClientAdapter: &tqClient{Client: apiClient},
|
2016-12-12 00:55:14 +00:00
|
|
|
downloadAdapterFuncs: make(map[string]NewAdapterFunc),
|
|
|
|
uploadAdapterFuncs: make(map[string]NewAdapterFunc),
|
2021-03-17 18:19:20 +00:00
|
|
|
sshTransfer: sshTransfer,
|
2016-08-09 21:43:29 +00:00
|
|
|
}
|
|
|
|
|
2016-12-13 22:38:10 +00:00
|
|
|
var tusAllowed bool
|
2017-01-04 16:05:29 +00:00
|
|
|
if git := apiClient.GitEnv(); git != nil {
|
2016-12-13 22:38:10 +00:00
|
|
|
if v := git.Int("lfs.transfer.maxretries", 0); v > 0 {
|
2016-12-13 23:48:54 +00:00
|
|
|
m.maxRetries = v
|
2016-12-13 22:38:10 +00:00
|
|
|
}
|
2020-04-15 03:13:27 +00:00
|
|
|
if v := git.Int("lfs.transfer.maxretrydelay", -1); v > -1 {
|
|
|
|
m.maxRetryDelay = v
|
|
|
|
}
|
2016-12-13 22:38:10 +00:00
|
|
|
if v := git.Int("lfs.concurrenttransfers", 0); v > 0 {
|
2016-12-13 23:48:54 +00:00
|
|
|
m.concurrentTransfers = v
|
2016-12-13 22:38:10 +00:00
|
|
|
}
|
|
|
|
m.basicTransfersOnly = git.Bool("lfs.basictransfersonly", false)
|
2017-09-17 16:41:30 +00:00
|
|
|
m.standaloneTransferAgent = findStandaloneTransfer(
|
|
|
|
apiClient, operation, remote,
|
|
|
|
)
|
2016-12-13 22:38:10 +00:00
|
|
|
tusAllowed = git.Bool("lfs.tustransfers", false)
|
|
|
|
configureCustomAdapters(git, m)
|
|
|
|
}
|
|
|
|
|
2016-12-13 23:48:54 +00:00
|
|
|
if m.maxRetries < 1 {
|
|
|
|
m.maxRetries = defaultMaxRetries
|
2016-12-13 20:32:25 +00:00
|
|
|
}
|
2020-04-15 03:13:27 +00:00
|
|
|
if m.maxRetryDelay < 1 {
|
|
|
|
m.maxRetryDelay = defaultMaxRetryDelay
|
|
|
|
}
|
2016-12-13 23:48:54 +00:00
|
|
|
|
2017-01-04 21:46:30 +00:00
|
|
|
if m.concurrentTransfers < 1 {
|
2016-12-13 23:48:54 +00:00
|
|
|
m.concurrentTransfers = defaultConcurrentTransfers
|
2016-12-13 20:06:58 +00:00
|
|
|
}
|
|
|
|
|
2021-03-17 18:19:20 +00:00
|
|
|
if sshTransfer != nil {
|
|
|
|
// Multiple concurrent transfers are not yet supported.
|
|
|
|
m.batchClientAdapter = &SSHBatchClient{
|
|
|
|
maxRetries: m.maxRetries,
|
|
|
|
transfer: sshTransfer,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-09 21:43:29 +00:00
|
|
|
configureBasicDownloadAdapter(m)
|
|
|
|
configureBasicUploadAdapter(m)
|
2016-12-13 22:38:10 +00:00
|
|
|
if tusAllowed {
|
2016-08-09 21:43:29 +00:00
|
|
|
configureTusAdapter(m)
|
|
|
|
}
|
2021-03-17 18:19:20 +00:00
|
|
|
configureSSHAdapter(m)
|
2016-12-14 16:53:02 +00:00
|
|
|
return m
|
2016-12-13 21:14:30 +00:00
|
|
|
}
|
|
|
|
|
2019-07-31 13:46:39 +00:00
|
|
|
func findDefaultStandaloneTransfer(url string) string {
|
|
|
|
if strings.HasPrefix(url, "file://") {
|
|
|
|
return standaloneFileName
|
|
|
|
}
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
2017-09-17 16:41:30 +00:00
|
|
|
func findStandaloneTransfer(client *lfsapi.Client, operation, remote string) string {
|
|
|
|
if operation == "" || remote == "" {
|
|
|
|
v, _ := client.GitEnv().Get("lfs.standalonetransferagent")
|
|
|
|
return v
|
|
|
|
}
|
|
|
|
|
2021-03-17 18:19:20 +00:00
|
|
|
ep := client.Endpoints.Endpoint(operation, remote)
|
2019-11-06 22:23:22 +00:00
|
|
|
aep := client.Endpoints.Endpoint(operation, remote)
|
2017-09-17 16:41:30 +00:00
|
|
|
uc := config.NewURLConfig(client.GitEnv())
|
|
|
|
v, ok := uc.Get("lfs", ep.Url, "standalonetransferagent")
|
|
|
|
if !ok {
|
2019-11-06 22:23:22 +00:00
|
|
|
return findDefaultStandaloneTransfer(aep.Url)
|
2017-09-17 16:41:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return v
|
|
|
|
}
|
|
|
|
|
2016-08-09 21:43:29 +00:00
|
|
|
// GetAdapterNames returns a list of the names of adapters available to be created
|
|
|
|
func (m *Manifest) GetAdapterNames(dir Direction) []string {
|
|
|
|
switch dir {
|
|
|
|
case Upload:
|
|
|
|
return m.GetUploadAdapterNames()
|
|
|
|
case Download:
|
|
|
|
return m.GetDownloadAdapterNames()
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetDownloadAdapterNames returns a list of the names of download adapters available to be created
|
|
|
|
func (m *Manifest) GetDownloadAdapterNames() []string {
|
|
|
|
return m.getAdapterNames(m.downloadAdapterFuncs)
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetUploadAdapterNames returns a list of the names of upload adapters available to be created
|
|
|
|
func (m *Manifest) GetUploadAdapterNames() []string {
|
|
|
|
return m.getAdapterNames(m.uploadAdapterFuncs)
|
|
|
|
}
|
|
|
|
|
2016-08-10 14:39:57 +00:00
|
|
|
// getAdapterNames returns a list of the names of adapters available to be created
|
2016-12-12 00:55:14 +00:00
|
|
|
func (m *Manifest) getAdapterNames(adapters map[string]NewAdapterFunc) []string {
|
2016-12-13 22:38:10 +00:00
|
|
|
if m.basicTransfersOnly {
|
2016-08-09 21:43:29 +00:00
|
|
|
return []string{BasicAdapterName}
|
|
|
|
}
|
|
|
|
|
|
|
|
m.mu.Lock()
|
|
|
|
defer m.mu.Unlock()
|
|
|
|
|
|
|
|
ret := make([]string, 0, len(adapters))
|
|
|
|
for n, _ := range adapters {
|
|
|
|
ret = append(ret, n)
|
|
|
|
}
|
|
|
|
return ret
|
|
|
|
}
|
|
|
|
|
|
|
|
// RegisterNewTransferAdapterFunc registers a new function for creating upload
|
|
|
|
// or download adapters. If a function with that name & direction is already
|
|
|
|
// registered, it is overridden
|
2016-12-12 00:55:14 +00:00
|
|
|
func (m *Manifest) RegisterNewAdapterFunc(name string, dir Direction, f NewAdapterFunc) {
|
2016-08-09 21:43:29 +00:00
|
|
|
m.mu.Lock()
|
|
|
|
defer m.mu.Unlock()
|
|
|
|
|
|
|
|
switch dir {
|
|
|
|
case Upload:
|
|
|
|
m.uploadAdapterFuncs[name] = f
|
|
|
|
case Download:
|
|
|
|
m.downloadAdapterFuncs[name] = f
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a new adapter by name and direction; default to BasicAdapterName if doesn't exist
|
2016-12-12 00:55:14 +00:00
|
|
|
func (m *Manifest) NewAdapterOrDefault(name string, dir Direction) Adapter {
|
2016-08-09 21:43:29 +00:00
|
|
|
if len(name) == 0 {
|
|
|
|
name = BasicAdapterName
|
|
|
|
}
|
|
|
|
|
|
|
|
a := m.NewAdapter(name, dir)
|
|
|
|
if a == nil {
|
|
|
|
tracerx.Printf("Defaulting to basic transfer adapter since %q did not exist", name)
|
|
|
|
a = m.NewAdapter(BasicAdapterName, dir)
|
|
|
|
}
|
|
|
|
return a
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a new adapter by name and direction, or nil if doesn't exist
|
2016-12-12 00:55:14 +00:00
|
|
|
func (m *Manifest) NewAdapter(name string, dir Direction) Adapter {
|
2016-08-09 21:43:29 +00:00
|
|
|
m.mu.Lock()
|
|
|
|
defer m.mu.Unlock()
|
|
|
|
|
|
|
|
switch dir {
|
|
|
|
case Upload:
|
|
|
|
if u, ok := m.uploadAdapterFuncs[name]; ok {
|
|
|
|
return u(name, dir)
|
|
|
|
}
|
|
|
|
case Download:
|
|
|
|
if d, ok := m.downloadAdapterFuncs[name]; ok {
|
|
|
|
return d(name, dir)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a new download adapter by name, or BasicAdapterName if doesn't exist
|
2016-12-12 00:55:14 +00:00
|
|
|
func (m *Manifest) NewDownloadAdapter(name string) Adapter {
|
2016-08-09 21:43:29 +00:00
|
|
|
return m.NewAdapterOrDefault(name, Download)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a new upload adapter by name, or BasicAdapterName if doesn't exist
|
2016-12-12 00:55:14 +00:00
|
|
|
func (m *Manifest) NewUploadAdapter(name string) Adapter {
|
2016-08-09 21:43:29 +00:00
|
|
|
return m.NewAdapterOrDefault(name, Upload)
|
|
|
|
}
|
2016-12-13 19:59:16 +00:00
|
|
|
|
2016-12-13 22:40:47 +00:00
|
|
|
// Env is any object with a config.Environment interface.
|
|
|
|
type Env interface {
|
2016-12-13 19:59:16 +00:00
|
|
|
Get(key string) (val string, ok bool)
|
2017-04-12 21:29:11 +00:00
|
|
|
GetAll(key string) []string
|
2016-12-13 19:59:16 +00:00
|
|
|
Bool(key string, def bool) (val bool)
|
|
|
|
Int(key string, def int) (val int)
|
2017-04-12 21:29:11 +00:00
|
|
|
All() map[string][]string
|
2016-12-13 19:59:16 +00:00
|
|
|
}
|