Merge branch 'master' into config-next-no-git-immut
This commit is contained in:
commit
3c3a0b3b8d
@ -11,6 +11,7 @@ import (
|
||||
"github.com/github/git-lfs/git"
|
||||
"github.com/github/git-lfs/lfs"
|
||||
"github.com/github/git-lfs/progress"
|
||||
"github.com/github/git-lfs/transfer"
|
||||
"github.com/rubyist/tracerx"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@ -175,6 +176,9 @@ func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
|
||||
// locked state.
|
||||
|
||||
// As files come in, write them to the wd and update the index
|
||||
|
||||
manifest := transfer.ConfigureManifest(transfer.NewManifest(), cfg)
|
||||
|
||||
for pointer := range in {
|
||||
|
||||
// Check the content - either missing or still this pointer (not exist is ok)
|
||||
@ -197,7 +201,7 @@ func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
|
||||
repopathchan <- pointer.Name
|
||||
cwdfilepath := <-cwdpathchan
|
||||
|
||||
err = lfs.PointerSmudgeToFile(cwdfilepath, pointer.Pointer, false, nil)
|
||||
err = lfs.PointerSmudgeToFile(cwdfilepath, pointer.Pointer, false, manifest, nil)
|
||||
if err != nil {
|
||||
if errutil.IsDownloadDeclinedError(err) {
|
||||
// acceptable error, data not local (fetch not run or include/exclude)
|
||||
|
@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/github/git-lfs/errutil"
|
||||
"github.com/github/git-lfs/lfs"
|
||||
"github.com/github/git-lfs/transfer"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@ -67,7 +68,8 @@ func smudgeCommand(cmd *cobra.Command, args []string) {
|
||||
download = false
|
||||
}
|
||||
|
||||
err = ptr.Smudge(os.Stdout, filename, download, cb)
|
||||
manifest := transfer.ConfigureManifest(transfer.NewManifest(), cfg)
|
||||
err = ptr.Smudge(os.Stdout, filename, download, manifest, cb)
|
||||
if file != nil {
|
||||
file.Close()
|
||||
}
|
||||
|
@ -68,11 +68,12 @@ func ObjectExistsOfSize(oid string, size int64) bool {
|
||||
}
|
||||
|
||||
func Environ() []string {
|
||||
manifest := transfer.ConfigureManifest(transfer.NewManifest(), config.Config)
|
||||
osEnviron := os.Environ()
|
||||
env := make([]string, 0, len(osEnviron)+7)
|
||||
dltransfers := transfer.GetDownloadAdapterNames()
|
||||
dltransfers := manifest.GetDownloadAdapterNames()
|
||||
sort.Strings(dltransfers)
|
||||
ultransfers := transfer.GetUploadAdapterNames()
|
||||
ultransfers := manifest.GetUploadAdapterNames()
|
||||
sort.Strings(ultransfers)
|
||||
|
||||
env = append(env,
|
||||
|
@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/github/git-lfs/errutil"
|
||||
"github.com/github/git-lfs/progress"
|
||||
"github.com/github/git-lfs/transfer"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -60,8 +61,8 @@ func NewPointerExtension(name string, priority int, oid string) *PointerExtensio
|
||||
return &PointerExtension{name, priority, oid, oidType}
|
||||
}
|
||||
|
||||
func (p *Pointer) Smudge(writer io.Writer, workingfile string, download bool, cb progress.CopyCallback) error {
|
||||
return PointerSmudge(writer, p, workingfile, download, cb)
|
||||
func (p *Pointer) Smudge(writer io.Writer, workingfile string, download bool, manifest *transfer.Manifest, cb progress.CopyCallback) error {
|
||||
return PointerSmudge(writer, p, workingfile, download, manifest, cb)
|
||||
}
|
||||
|
||||
func (p *Pointer) Encode(writer io.Writer) (int, error) {
|
||||
|
@ -17,14 +17,14 @@ import (
|
||||
"github.com/rubyist/tracerx"
|
||||
)
|
||||
|
||||
func PointerSmudgeToFile(filename string, ptr *Pointer, download bool, cb progress.CopyCallback) error {
|
||||
func PointerSmudgeToFile(filename string, ptr *Pointer, download bool, manifest *transfer.Manifest, cb progress.CopyCallback) error {
|
||||
os.MkdirAll(filepath.Dir(filename), 0755)
|
||||
file, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not create working directory file: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
if err := PointerSmudge(file, ptr, filename, download, cb); err != nil {
|
||||
if err := PointerSmudge(file, ptr, filename, download, manifest, cb); err != nil {
|
||||
if errutil.IsDownloadDeclinedError(err) {
|
||||
// write placeholder data instead
|
||||
file.Seek(0, os.SEEK_SET)
|
||||
@ -37,7 +37,7 @@ func PointerSmudgeToFile(filename string, ptr *Pointer, download bool, cb progre
|
||||
return nil
|
||||
}
|
||||
|
||||
func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download bool, cb progress.CopyCallback) error {
|
||||
func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download bool, manifest *transfer.Manifest, cb progress.CopyCallback) error {
|
||||
mediafile, err := LocalMediaPath(ptr.Oid)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -58,7 +58,7 @@ func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download
|
||||
|
||||
if statErr != nil || stat == nil {
|
||||
if download {
|
||||
err = downloadFile(writer, ptr, workingfile, mediafile, cb)
|
||||
err = downloadFile(writer, ptr, workingfile, mediafile, manifest, cb)
|
||||
} else {
|
||||
return errutil.NewDownloadDeclinedError(nil)
|
||||
}
|
||||
@ -73,10 +73,10 @@ func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download
|
||||
return nil
|
||||
}
|
||||
|
||||
func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, cb progress.CopyCallback) error {
|
||||
func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, manifest *transfer.Manifest, cb progress.CopyCallback) error {
|
||||
fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, pb.FormatBytes(ptr.Size))
|
||||
|
||||
xfers := transfer.GetDownloadAdapterNames()
|
||||
xfers := manifest.GetDownloadAdapterNames()
|
||||
obj, adapterName, err := api.BatchOrLegacySingle(config.Config, &api.ObjectResource{Oid: ptr.Oid, Size: ptr.Size}, "download", xfers)
|
||||
if err != nil {
|
||||
return errutil.Errorf(err, "Error downloading %s: %s", filepath.Base(mediafile), err)
|
||||
@ -86,7 +86,7 @@ func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string,
|
||||
ptr.Size = obj.Size
|
||||
}
|
||||
|
||||
adapter := transfer.NewDownloadAdapter(adapterName)
|
||||
adapter := manifest.NewDownloadAdapter(adapterName)
|
||||
var tcb transfer.TransferProgressCallback
|
||||
if cb != nil {
|
||||
tcb = func(name string, totalSize, readSoFar int64, readSinceLast int) error {
|
||||
|
@ -53,6 +53,7 @@ type TransferQueue struct {
|
||||
retrywait sync.WaitGroup
|
||||
wait sync.WaitGroup // Incremented on Add(), decremented on transfer complete or skip
|
||||
oldApiWorkers int // Number of non-batch API workers to spawn (deprecated)
|
||||
manifest *transfer.Manifest
|
||||
}
|
||||
|
||||
// newTransferQueue builds a TransferQueue, direction and underlying mechanism determined by adapter
|
||||
@ -67,6 +68,7 @@ func newTransferQueue(files int, size int64, dryRun bool, dir transfer.Direction
|
||||
oldApiWorkers: config.Config.ConcurrentTransfers(),
|
||||
transferables: make(map[string]Transferable),
|
||||
trMutex: &sync.Mutex{},
|
||||
manifest: transfer.ConfigureManifest(transfer.NewManifest(), config.Config),
|
||||
}
|
||||
|
||||
q.errorwait.Add(1)
|
||||
@ -107,7 +109,7 @@ func (q *TransferQueue) useAdapter(name string) {
|
||||
// changing adapter support in between batches
|
||||
q.finishAdapter()
|
||||
}
|
||||
q.adapter = transfer.NewAdapterOrDefault(name, q.direction)
|
||||
q.adapter = q.manifest.NewAdapterOrDefault(name, q.direction)
|
||||
}
|
||||
|
||||
func (q *TransferQueue) finishAdapter() {
|
||||
@ -330,7 +332,7 @@ func (q *TransferQueue) legacyFallback(failedBatch []interface{}) {
|
||||
func (q *TransferQueue) batchApiRoutine() {
|
||||
var startProgress sync.Once
|
||||
|
||||
transferAdapterNames := transfer.GetAdapterNames(q.direction)
|
||||
transferAdapterNames := q.manifest.GetAdapterNames(q.direction)
|
||||
|
||||
for {
|
||||
batch := q.batcher.Next()
|
||||
|
@ -214,11 +214,10 @@ func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback
|
||||
}
|
||||
|
||||
return tools.RenameFileCopyPermissions(dlfilename, t.Path)
|
||||
|
||||
}
|
||||
|
||||
func init() {
|
||||
newfunc := func(name string, dir Direction) TransferAdapter {
|
||||
func configureBasicDownloadAdapter(m *Manifest) {
|
||||
m.RegisterNewTransferAdapterFunc(BasicAdapterName, Download, func(name string, dir Direction) TransferAdapter {
|
||||
switch dir {
|
||||
case Download:
|
||||
bd := &basicDownloadAdapter{newAdapterBase(name, dir, nil)}
|
||||
@ -229,6 +228,5 @@ func init() {
|
||||
panic("Should never ask this func to upload")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
RegisterNewTransferAdapterFunc(BasicAdapterName, Download, newfunc)
|
||||
})
|
||||
}
|
||||
|
@ -138,8 +138,8 @@ func newStartCallbackReader(r io.Reader, cb func(*startCallbackReader)) *startCa
|
||||
return &startCallbackReader{r, cb, false}
|
||||
}
|
||||
|
||||
func init() {
|
||||
newfunc := func(name string, dir Direction) TransferAdapter {
|
||||
func configureBasicUploadAdapter(m *Manifest) {
|
||||
m.RegisterNewTransferAdapterFunc(BasicAdapterName, Upload, func(name string, dir Direction) TransferAdapter {
|
||||
switch dir {
|
||||
case Upload:
|
||||
bu := &basicUploadAdapter{newAdapterBase(name, dir, nil)}
|
||||
@ -150,6 +150,5 @@ func init() {
|
||||
panic("Should never ask this func for basic download")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
RegisterNewTransferAdapterFunc(BasicAdapterName, Upload, newfunc)
|
||||
})
|
||||
}
|
||||
|
@ -347,9 +347,9 @@ func newCustomAdapter(name string, dir Direction, path, args string, concurrent
|
||||
}
|
||||
|
||||
// Initialise custom adapters based on current config
|
||||
func ConfigureCustomAdapters() {
|
||||
func configureCustomAdapters(cfg *config.Configuration, m *Manifest) {
|
||||
pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
|
||||
for k, v := range config.Config.AllGitConfig() {
|
||||
for k, v := range cfg.AllGitConfig() {
|
||||
match := pathRegex.FindStringSubmatch(k)
|
||||
if match == nil {
|
||||
continue
|
||||
@ -358,9 +358,9 @@ func ConfigureCustomAdapters() {
|
||||
name := match[1]
|
||||
path := v
|
||||
// retrieve other values
|
||||
args, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
|
||||
concurrent := config.Config.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
|
||||
direction, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
|
||||
args, _ := cfg.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
|
||||
concurrent := cfg.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
|
||||
direction, _ := cfg.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
|
||||
if len(direction) == 0 {
|
||||
direction = "both"
|
||||
} else {
|
||||
@ -373,10 +373,10 @@ func ConfigureCustomAdapters() {
|
||||
}
|
||||
|
||||
if direction == "download" || direction == "both" {
|
||||
RegisterNewTransferAdapterFunc(name, Download, newfunc)
|
||||
m.RegisterNewTransferAdapterFunc(name, Download, newfunc)
|
||||
}
|
||||
if direction == "upload" || direction == "both" {
|
||||
RegisterNewTransferAdapterFunc(name, Upload, newfunc)
|
||||
m.RegisterNewTransferAdapterFunc(name, Upload, newfunc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -3,58 +3,19 @@ package transfer
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/github/git-lfs/config"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var (
|
||||
savedDownloadAdapterFuncs map[string]NewTransferAdapterFunc
|
||||
savedUploadAdapterFuncs map[string]NewTransferAdapterFunc
|
||||
)
|
||||
|
||||
func copyFuncMap(to, from map[string]NewTransferAdapterFunc) {
|
||||
for k, v := range from {
|
||||
to[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
func saveTransferSetupState() {
|
||||
funcMutex.Lock()
|
||||
defer funcMutex.Unlock()
|
||||
|
||||
savedDownloadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
|
||||
copyFuncMap(savedDownloadAdapterFuncs, downloadAdapterFuncs)
|
||||
|
||||
savedUploadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
|
||||
copyFuncMap(savedUploadAdapterFuncs, uploadAdapterFuncs)
|
||||
}
|
||||
|
||||
func restoreTransferSetupState() {
|
||||
funcMutex.Lock()
|
||||
defer funcMutex.Unlock()
|
||||
|
||||
downloadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
|
||||
copyFuncMap(downloadAdapterFuncs, savedDownloadAdapterFuncs)
|
||||
|
||||
uploadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
|
||||
copyFuncMap(uploadAdapterFuncs, savedUploadAdapterFuncs)
|
||||
|
||||
}
|
||||
|
||||
func TestCustomTransferBasicConfig(t *testing.T) {
|
||||
saveTransferSetupState()
|
||||
defer func() {
|
||||
config.Config.ResetConfig()
|
||||
restoreTransferSetupState()
|
||||
}()
|
||||
|
||||
path := "/path/to/binary"
|
||||
config.Config.SetConfig("lfs.customtransfer.testsimple.path", path)
|
||||
cfg := config.NewFrom(config.Values{
|
||||
Git: map[string]string{"lfs.customtransfer.testsimple.path": path},
|
||||
})
|
||||
|
||||
ConfigureCustomAdapters()
|
||||
m := ConfigureManifest(NewManifest(), cfg)
|
||||
|
||||
u := NewUploadAdapter("testsimple")
|
||||
u := m.NewUploadAdapter("testsimple")
|
||||
assert.NotNil(t, u, "Upload adapter should be present")
|
||||
cu, _ := u.(*customAdapter)
|
||||
assert.NotNil(t, cu, "Upload adapter should be customAdapter")
|
||||
@ -62,7 +23,7 @@ func TestCustomTransferBasicConfig(t *testing.T) {
|
||||
assert.Equal(t, cu.args, "", "args should be blank")
|
||||
assert.Equal(t, cu.concurrent, true, "concurrent should be defaulted")
|
||||
|
||||
d := NewDownloadAdapter("testsimple")
|
||||
d := m.NewDownloadAdapter("testsimple")
|
||||
assert.NotNil(t, d, "Download adapter should be present")
|
||||
cd, _ := u.(*customAdapter)
|
||||
assert.NotNil(t, cd, "Download adapter should be customAdapter")
|
||||
@ -72,27 +33,25 @@ func TestCustomTransferBasicConfig(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCustomTransferDownloadConfig(t *testing.T) {
|
||||
saveTransferSetupState()
|
||||
defer func() {
|
||||
config.Config.ResetConfig()
|
||||
restoreTransferSetupState()
|
||||
}()
|
||||
|
||||
path := "/path/to/binary"
|
||||
args := "-c 1 --whatever"
|
||||
config.Config.SetConfig("lfs.customtransfer.testdownload.path", path)
|
||||
config.Config.SetConfig("lfs.customtransfer.testdownload.args", args)
|
||||
config.Config.SetConfig("lfs.customtransfer.testdownload.concurrent", "false")
|
||||
config.Config.SetConfig("lfs.customtransfer.testdownload.direction", "download")
|
||||
cfg := config.NewFrom(config.Values{
|
||||
Git: map[string]string{
|
||||
"lfs.customtransfer.testdownload.path": path,
|
||||
"lfs.customtransfer.testdownload.args": args,
|
||||
"lfs.customtransfer.testdownload.concurrent": "false",
|
||||
"lfs.customtransfer.testdownload.direction": "download",
|
||||
},
|
||||
})
|
||||
|
||||
ConfigureCustomAdapters()
|
||||
m := ConfigureManifest(NewManifest(), cfg)
|
||||
|
||||
u := NewUploadAdapter("testdownload")
|
||||
u := m.NewUploadAdapter("testdownload")
|
||||
assert.NotNil(t, u, "Upload adapter should always be created")
|
||||
cu, _ := u.(*customAdapter)
|
||||
assert.Nil(t, cu, "Upload adapter should NOT be custom (default to basic)")
|
||||
|
||||
d := NewDownloadAdapter("testdownload")
|
||||
d := m.NewDownloadAdapter("testdownload")
|
||||
assert.NotNil(t, d, "Download adapter should be present")
|
||||
cd, _ := d.(*customAdapter)
|
||||
assert.NotNil(t, cd, "Download adapter should be customAdapter")
|
||||
@ -102,27 +61,25 @@ func TestCustomTransferDownloadConfig(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCustomTransferUploadConfig(t *testing.T) {
|
||||
saveTransferSetupState()
|
||||
defer func() {
|
||||
config.Config.ResetConfig()
|
||||
restoreTransferSetupState()
|
||||
}()
|
||||
|
||||
path := "/path/to/binary"
|
||||
args := "-c 1 --whatever"
|
||||
config.Config.SetConfig("lfs.customtransfer.testupload.path", path)
|
||||
config.Config.SetConfig("lfs.customtransfer.testupload.args", args)
|
||||
config.Config.SetConfig("lfs.customtransfer.testupload.concurrent", "false")
|
||||
config.Config.SetConfig("lfs.customtransfer.testupload.direction", "upload")
|
||||
cfg := config.NewFrom(config.Values{
|
||||
Git: map[string]string{
|
||||
"lfs.customtransfer.testupload.path": path,
|
||||
"lfs.customtransfer.testupload.args": args,
|
||||
"lfs.customtransfer.testupload.concurrent": "false",
|
||||
"lfs.customtransfer.testupload.direction": "upload",
|
||||
},
|
||||
})
|
||||
|
||||
ConfigureCustomAdapters()
|
||||
m := ConfigureManifest(NewManifest(), cfg)
|
||||
|
||||
d := NewDownloadAdapter("testupload")
|
||||
d := m.NewDownloadAdapter("testupload")
|
||||
assert.NotNil(t, d, "Download adapter should always be created")
|
||||
cd, _ := d.(*customAdapter)
|
||||
assert.Nil(t, cd, "Download adapter should NOT be custom (default to basic)")
|
||||
|
||||
u := NewUploadAdapter("testupload")
|
||||
u := m.NewUploadAdapter("testupload")
|
||||
assert.NotNil(t, u, "Upload adapter should be present")
|
||||
cu, _ := u.(*customAdapter)
|
||||
assert.NotNil(t, cu, "Upload adapter should be customAdapter")
|
||||
@ -132,22 +89,20 @@ func TestCustomTransferUploadConfig(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCustomTransferBothConfig(t *testing.T) {
|
||||
saveTransferSetupState()
|
||||
defer func() {
|
||||
config.Config.ResetConfig()
|
||||
restoreTransferSetupState()
|
||||
}()
|
||||
|
||||
path := "/path/to/binary"
|
||||
args := "-c 1 --whatever --yeah"
|
||||
config.Config.SetConfig("lfs.customtransfer.testboth.path", path)
|
||||
config.Config.SetConfig("lfs.customtransfer.testboth.args", args)
|
||||
config.Config.SetConfig("lfs.customtransfer.testboth.concurrent", "yes")
|
||||
config.Config.SetConfig("lfs.customtransfer.testboth.direction", "both")
|
||||
cfg := config.NewFrom(config.Values{
|
||||
Git: map[string]string{
|
||||
"lfs.customtransfer.testboth.path": path,
|
||||
"lfs.customtransfer.testboth.args": args,
|
||||
"lfs.customtransfer.testboth.concurrent": "yes",
|
||||
"lfs.customtransfer.testboth.direction": "both",
|
||||
},
|
||||
})
|
||||
|
||||
ConfigureCustomAdapters()
|
||||
m := ConfigureManifest(NewManifest(), cfg)
|
||||
|
||||
d := NewDownloadAdapter("testboth")
|
||||
d := m.NewDownloadAdapter("testboth")
|
||||
assert.NotNil(t, d, "Download adapter should be present")
|
||||
cd, _ := d.(*customAdapter)
|
||||
assert.NotNil(t, cd, "Download adapter should be customAdapter")
|
||||
@ -155,7 +110,7 @@ func TestCustomTransferBothConfig(t *testing.T) {
|
||||
assert.Equal(t, cd.args, args, "args should be correct")
|
||||
assert.Equal(t, cd.concurrent, true, "concurrent should be set")
|
||||
|
||||
u := NewUploadAdapter("testboth")
|
||||
u := m.NewUploadAdapter("testboth")
|
||||
assert.NotNil(t, u, "Upload adapter should be present")
|
||||
cu, _ := u.(*customAdapter)
|
||||
assert.NotNil(t, cu, "Upload adapter should be customAdapter")
|
||||
|
128
transfer/manifest.go
Normal file
128
transfer/manifest.go
Normal file
@ -0,0 +1,128 @@
|
||||
package transfer
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/github/git-lfs/config"
|
||||
"github.com/rubyist/tracerx"
|
||||
)
|
||||
|
||||
type Manifest struct {
|
||||
basicTransfersOnly bool
|
||||
downloadAdapterFuncs map[string]NewTransferAdapterFunc
|
||||
uploadAdapterFuncs map[string]NewTransferAdapterFunc
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func NewManifest() *Manifest {
|
||||
return &Manifest{
|
||||
downloadAdapterFuncs: make(map[string]NewTransferAdapterFunc),
|
||||
uploadAdapterFuncs: make(map[string]NewTransferAdapterFunc),
|
||||
}
|
||||
}
|
||||
|
||||
func ConfigureManifest(m *Manifest, cfg *config.Configuration) *Manifest {
|
||||
m.basicTransfersOnly = cfg.BasicTransfersOnly()
|
||||
|
||||
configureBasicDownloadAdapter(m)
|
||||
configureBasicUploadAdapter(m)
|
||||
if cfg.TusTransfersAllowed() {
|
||||
configureTusAdapter(m)
|
||||
}
|
||||
configureCustomAdapters(cfg, m)
|
||||
return m
|
||||
}
|
||||
|
||||
// GetAdapterNames returns a list of the names of adapters available to be created
|
||||
func (m *Manifest) GetAdapterNames(dir Direction) []string {
|
||||
switch dir {
|
||||
case Upload:
|
||||
return m.GetUploadAdapterNames()
|
||||
case Download:
|
||||
return m.GetDownloadAdapterNames()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetDownloadAdapterNames returns a list of the names of download adapters available to be created
|
||||
func (m *Manifest) GetDownloadAdapterNames() []string {
|
||||
return m.getAdapterNames(m.downloadAdapterFuncs)
|
||||
}
|
||||
|
||||
// GetUploadAdapterNames returns a list of the names of upload adapters available to be created
|
||||
func (m *Manifest) GetUploadAdapterNames() []string {
|
||||
return m.getAdapterNames(m.uploadAdapterFuncs)
|
||||
}
|
||||
|
||||
// getAdapterNames returns a list of the names of adapters available to be created
|
||||
func (m *Manifest) getAdapterNames(adapters map[string]NewTransferAdapterFunc) []string {
|
||||
if m.basicTransfersOnly {
|
||||
return []string{BasicAdapterName}
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
ret := make([]string, 0, len(adapters))
|
||||
for n, _ := range adapters {
|
||||
ret = append(ret, n)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// RegisterNewTransferAdapterFunc registers a new function for creating upload
|
||||
// or download adapters. If a function with that name & direction is already
|
||||
// registered, it is overridden
|
||||
func (m *Manifest) RegisterNewTransferAdapterFunc(name string, dir Direction, f NewTransferAdapterFunc) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
switch dir {
|
||||
case Upload:
|
||||
m.uploadAdapterFuncs[name] = f
|
||||
case Download:
|
||||
m.downloadAdapterFuncs[name] = f
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new adapter by name and direction; default to BasicAdapterName if doesn't exist
|
||||
func (m *Manifest) NewAdapterOrDefault(name string, dir Direction) TransferAdapter {
|
||||
if len(name) == 0 {
|
||||
name = BasicAdapterName
|
||||
}
|
||||
|
||||
a := m.NewAdapter(name, dir)
|
||||
if a == nil {
|
||||
tracerx.Printf("Defaulting to basic transfer adapter since %q did not exist", name)
|
||||
a = m.NewAdapter(BasicAdapterName, dir)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// Create a new adapter by name and direction, or nil if doesn't exist
|
||||
func (m *Manifest) NewAdapter(name string, dir Direction) TransferAdapter {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
switch dir {
|
||||
case Upload:
|
||||
if u, ok := m.uploadAdapterFuncs[name]; ok {
|
||||
return u(name, dir)
|
||||
}
|
||||
case Download:
|
||||
if d, ok := m.downloadAdapterFuncs[name]; ok {
|
||||
return d(name, dir)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create a new download adapter by name, or BasicAdapterName if doesn't exist
|
||||
func (m *Manifest) NewDownloadAdapter(name string) TransferAdapter {
|
||||
return m.NewAdapterOrDefault(name, Download)
|
||||
}
|
||||
|
||||
// Create a new upload adapter by name, or BasicAdapterName if doesn't exist
|
||||
func (m *Manifest) NewUploadAdapter(name string) TransferAdapter {
|
||||
return m.NewAdapterOrDefault(name, Upload)
|
||||
}
|
@ -2,14 +2,7 @@
|
||||
// NOTE: Subject to change, do not rely on this package from outside git-lfs source
|
||||
package transfer
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/github/git-lfs/config"
|
||||
|
||||
"github.com/github/git-lfs/api"
|
||||
"github.com/rubyist/tracerx"
|
||||
)
|
||||
import "github.com/github/git-lfs/api"
|
||||
|
||||
type Direction int
|
||||
|
||||
@ -20,16 +13,10 @@ const (
|
||||
|
||||
// NewTransferAdapterFunc creates new instances of TransferAdapter. Code that wishes
|
||||
// to provide new TransferAdapter instances should pass an implementation of this
|
||||
// function to RegisterNewTransferAdapterFunc
|
||||
// function to RegisterNewTransferAdapterFunc() on a *Manifest.
|
||||
// name and dir are to provide context if one func implements many instances
|
||||
type NewTransferAdapterFunc func(name string, dir Direction) TransferAdapter
|
||||
|
||||
var (
|
||||
funcMutex sync.Mutex
|
||||
downloadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
|
||||
uploadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
|
||||
)
|
||||
|
||||
type TransferProgressCallback func(name string, totalSize, readSoFar int64, readSinceLast int) error
|
||||
|
||||
// TransferAdapter is implemented by types which can upload and/or download LFS
|
||||
@ -92,128 +79,3 @@ type TransferResult struct {
|
||||
// This will be non-nil if there was an error transferring this item
|
||||
Error error
|
||||
}
|
||||
|
||||
// GetAdapterNames returns a list of the names of adapters available to be created
|
||||
func GetAdapterNames(dir Direction) []string {
|
||||
switch dir {
|
||||
case Upload:
|
||||
return GetUploadAdapterNames()
|
||||
case Download:
|
||||
return GetDownloadAdapterNames()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetDownloadAdapterNames returns a list of the names of download adapters available to be created
|
||||
func GetDownloadAdapterNames() []string {
|
||||
|
||||
if config.Config.BasicTransfersOnly() {
|
||||
return []string{BasicAdapterName}
|
||||
}
|
||||
|
||||
initCoreAdaptersIfRequired()
|
||||
|
||||
funcMutex.Lock()
|
||||
defer funcMutex.Unlock()
|
||||
|
||||
ret := make([]string, 0, len(downloadAdapterFuncs))
|
||||
for n, _ := range downloadAdapterFuncs {
|
||||
ret = append(ret, n)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// GetUploadAdapterNames returns a list of the names of upload adapters available to be created
|
||||
func GetUploadAdapterNames() []string {
|
||||
|
||||
if config.Config.BasicTransfersOnly() {
|
||||
return []string{BasicAdapterName}
|
||||
}
|
||||
|
||||
initCoreAdaptersIfRequired()
|
||||
|
||||
funcMutex.Lock()
|
||||
defer funcMutex.Unlock()
|
||||
|
||||
ret := make([]string, 0, len(uploadAdapterFuncs))
|
||||
for n, _ := range uploadAdapterFuncs {
|
||||
ret = append(ret, n)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// RegisterNewTransferAdapterFunc registers a new function for creating upload
|
||||
// or download adapters. If a function with that name & direction is already
|
||||
// registered, it is overridden
|
||||
func RegisterNewTransferAdapterFunc(name string, dir Direction, f NewTransferAdapterFunc) {
|
||||
funcMutex.Lock()
|
||||
defer funcMutex.Unlock()
|
||||
|
||||
switch dir {
|
||||
case Upload:
|
||||
uploadAdapterFuncs[name] = f
|
||||
case Download:
|
||||
downloadAdapterFuncs[name] = f
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new adapter by name and direction; default to BasicAdapterName if doesn't exist
|
||||
func NewAdapterOrDefault(name string, dir Direction) TransferAdapter {
|
||||
if len(name) == 0 {
|
||||
name = BasicAdapterName
|
||||
}
|
||||
|
||||
a := NewAdapter(name, dir)
|
||||
if a == nil {
|
||||
tracerx.Printf("Defaulting to basic transfer adapter since %q did not exist", name)
|
||||
a = NewAdapter(BasicAdapterName, dir)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// Create a new adapter by name and direction, or nil if doesn't exist
|
||||
func NewAdapter(name string, dir Direction) TransferAdapter {
|
||||
initCoreAdaptersIfRequired()
|
||||
|
||||
funcMutex.Lock()
|
||||
defer funcMutex.Unlock()
|
||||
|
||||
switch dir {
|
||||
case Upload:
|
||||
if u, ok := uploadAdapterFuncs[name]; ok {
|
||||
return u(name, dir)
|
||||
}
|
||||
case Download:
|
||||
if d, ok := downloadAdapterFuncs[name]; ok {
|
||||
return d(name, dir)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create a new download adapter by name, or BasicAdapterName if doesn't exist
|
||||
func NewDownloadAdapter(name string) TransferAdapter {
|
||||
return NewAdapterOrDefault(name, Download)
|
||||
}
|
||||
|
||||
// Create a new upload adapter by name, or BasicAdapterName if doesn't exist
|
||||
func NewUploadAdapter(name string) TransferAdapter {
|
||||
return NewAdapterOrDefault(name, Upload)
|
||||
}
|
||||
|
||||
var initCoreOnce sync.Once
|
||||
|
||||
func initCoreAdaptersIfRequired() {
|
||||
// It's important to late-init custom adapters because they rely on Config
|
||||
// And if we cause Config to load too early it causes issues
|
||||
// That's why this isn't in an init() block
|
||||
initCoreOnce.Do(func() {
|
||||
ConfigureCustomAdapters()
|
||||
|
||||
// tus.io upload adapter is still experimental, requires
|
||||
// `lfs.tustransfers=true` to activate.
|
||||
if !config.Config.TusTransfersAllowed() {
|
||||
delete(uploadAdapterFuncs, TusAdapterName)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
@ -16,47 +16,55 @@ type testAdapter struct {
|
||||
func (a *testAdapter) Name() string {
|
||||
return a.name
|
||||
}
|
||||
|
||||
func (a *testAdapter) Direction() Direction {
|
||||
return a.dir
|
||||
}
|
||||
|
||||
func (a *testAdapter) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *testAdapter) Add(t *Transfer) {
|
||||
}
|
||||
|
||||
func (a *testAdapter) End() {
|
||||
}
|
||||
|
||||
func (a *testAdapter) ClearTempStorage() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func newTestAdapter(name string, dir Direction) TransferAdapter {
|
||||
return &testAdapter{name, dir}
|
||||
}
|
||||
|
||||
func newRenamedTestAdapter(name string, dir Direction) TransferAdapter {
|
||||
return &testAdapter{"RENAMED", dir}
|
||||
}
|
||||
func resetAdapters() {
|
||||
uploadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
|
||||
downloadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
|
||||
}
|
||||
|
||||
func testBasicAdapterExists(t *testing.T) {
|
||||
cfg := config.New()
|
||||
m := ConfigureManifest(NewManifest(), cfg)
|
||||
|
||||
assert := assert.New(t)
|
||||
|
||||
dls := GetDownloadAdapterNames()
|
||||
dls := m.GetDownloadAdapterNames()
|
||||
if assert.NotNil(dls) {
|
||||
assert.Equal([]string{"basic"}, dls)
|
||||
}
|
||||
uls := GetUploadAdapterNames()
|
||||
uls := m.GetUploadAdapterNames()
|
||||
if assert.NotNil(uls) {
|
||||
assert.Equal([]string{"basic"}, uls)
|
||||
}
|
||||
da := NewDownloadAdapter("basic")
|
||||
|
||||
da := m.NewDownloadAdapter("basic")
|
||||
if assert.NotNil(da) {
|
||||
assert.Equal("basic", da.Name())
|
||||
assert.Equal(Download, da.Direction())
|
||||
}
|
||||
ua := NewUploadAdapter("basic")
|
||||
|
||||
ua := m.NewUploadAdapter("basic")
|
||||
if assert.NotNil(ua) {
|
||||
assert.Equal("basic", ua.Name())
|
||||
assert.Equal(Upload, ua.Direction())
|
||||
@ -64,61 +72,69 @@ func testBasicAdapterExists(t *testing.T) {
|
||||
}
|
||||
|
||||
func testAdapterRegAndOverride(t *testing.T) {
|
||||
cfg := config.New()
|
||||
m := ConfigureManifest(NewManifest(), cfg)
|
||||
assert := assert.New(t)
|
||||
|
||||
assert.Nil(NewDownloadAdapter("test"))
|
||||
assert.Nil(NewUploadAdapter("test"))
|
||||
assert.Nil(m.NewDownloadAdapter("test"))
|
||||
assert.Nil(m.NewUploadAdapter("test"))
|
||||
|
||||
RegisterNewTransferAdapterFunc("test", Upload, newTestAdapter)
|
||||
assert.Nil(NewDownloadAdapter("test"))
|
||||
assert.NotNil(NewUploadAdapter("test"))
|
||||
m.RegisterNewTransferAdapterFunc("test", Upload, newTestAdapter)
|
||||
assert.Nil(m.NewDownloadAdapter("test"))
|
||||
assert.NotNil(m.NewUploadAdapter("test"))
|
||||
|
||||
RegisterNewTransferAdapterFunc("test", Download, newTestAdapter)
|
||||
da := NewDownloadAdapter("test")
|
||||
m.RegisterNewTransferAdapterFunc("test", Download, newTestAdapter)
|
||||
da := m.NewDownloadAdapter("test")
|
||||
if assert.NotNil(da) {
|
||||
assert.Equal("test", da.Name())
|
||||
assert.Equal(Download, da.Direction())
|
||||
}
|
||||
ua := NewUploadAdapter("test")
|
||||
|
||||
ua := m.NewUploadAdapter("test")
|
||||
if assert.NotNil(ua) {
|
||||
assert.Equal("test", ua.Name())
|
||||
assert.Equal(Upload, ua.Direction())
|
||||
}
|
||||
|
||||
// Test override
|
||||
RegisterNewTransferAdapterFunc("test", Upload, newRenamedTestAdapter)
|
||||
ua = NewUploadAdapter("test")
|
||||
m.RegisterNewTransferAdapterFunc("test", Upload, newRenamedTestAdapter)
|
||||
ua = m.NewUploadAdapter("test")
|
||||
if assert.NotNil(ua) {
|
||||
assert.Equal("RENAMED", ua.Name())
|
||||
assert.Equal(Upload, ua.Direction())
|
||||
}
|
||||
da = NewDownloadAdapter("test")
|
||||
|
||||
da = m.NewDownloadAdapter("test")
|
||||
if assert.NotNil(da) {
|
||||
assert.Equal("test", da.Name())
|
||||
assert.Equal(Download, da.Direction())
|
||||
}
|
||||
RegisterNewTransferAdapterFunc("test", Download, newRenamedTestAdapter)
|
||||
da = NewDownloadAdapter("test")
|
||||
|
||||
m.RegisterNewTransferAdapterFunc("test", Download, newRenamedTestAdapter)
|
||||
da = m.NewDownloadAdapter("test")
|
||||
if assert.NotNil(da) {
|
||||
assert.Equal("RENAMED", da.Name())
|
||||
assert.Equal(Download, da.Direction())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func testAdapterRegButBasicOnly(t *testing.T) {
|
||||
cfg := config.NewFrom(config.Values{
|
||||
Git: map[string]string{"lfs.basictransfersonly": "yes"},
|
||||
})
|
||||
m := ConfigureManifest(NewManifest(), cfg)
|
||||
|
||||
assert := assert.New(t)
|
||||
|
||||
config.Config.SetConfig("lfs.basictransfersonly", "yes")
|
||||
RegisterNewTransferAdapterFunc("test", Upload, newTestAdapter)
|
||||
RegisterNewTransferAdapterFunc("test", Download, newTestAdapter)
|
||||
m.RegisterNewTransferAdapterFunc("test", Upload, newTestAdapter)
|
||||
m.RegisterNewTransferAdapterFunc("test", Download, newTestAdapter)
|
||||
// Will still be created if we ask for them
|
||||
assert.NotNil(NewUploadAdapter("test"))
|
||||
assert.NotNil(NewDownloadAdapter("test"))
|
||||
assert.NotNil(m.NewUploadAdapter("test"))
|
||||
assert.NotNil(m.NewDownloadAdapter("test"))
|
||||
|
||||
// But list will exclude
|
||||
ld := GetDownloadAdapterNames()
|
||||
ld := m.GetDownloadAdapterNames()
|
||||
assert.Equal([]string{BasicAdapterName}, ld)
|
||||
lu := GetUploadAdapterNames()
|
||||
lu := m.GetUploadAdapterNames()
|
||||
assert.Equal([]string{BasicAdapterName}, lu)
|
||||
}
|
||||
|
@ -156,8 +156,8 @@ func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferP
|
||||
return api.VerifyUpload(config.Config, t.Object)
|
||||
}
|
||||
|
||||
func init() {
|
||||
newfunc := func(name string, dir Direction) TransferAdapter {
|
||||
func configureTusAdapter(m *Manifest) {
|
||||
m.RegisterNewTransferAdapterFunc(TusAdapterName, Upload, func(name string, dir Direction) TransferAdapter {
|
||||
switch dir {
|
||||
case Upload:
|
||||
bu := &tusUploadAdapter{newAdapterBase(name, dir, nil)}
|
||||
@ -168,6 +168,5 @@ func init() {
|
||||
panic("Should never ask tus.io to download")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
RegisterNewTransferAdapterFunc(TusAdapterName, Upload, newfunc)
|
||||
})
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user