lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into config-next-no-git-immut

Browse Source
This commit is contained in:
Taylor Blau 2016-08-10 12:53:57 -06:00
commit 3c3a0b3b8d
14 changed files with 256 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
commands/command_checkout.go
View File

@ -11,6 +11,7 @@ import (
"github.com/github/git-lfs/git"
"github.com/github/git-lfs/lfs"
"github.com/github/git-lfs/progress"
"github.com/github/git-lfs/transfer"
"github.com/rubyist/tracerx"
"github.com/spf13/cobra"
)
@ -175,6 +176,9 @@ func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
// locked state.
// As files come in, write them to the wd and update the index
manifest := transfer.ConfigureManifest(transfer.NewManifest(), cfg)
for pointer := range in {
// Check the content - either missing or still this pointer (not exist is ok)
@ -197,7 +201,7 @@ func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
repopathchan <- pointer.Name
cwdfilepath := <-cwdpathchan
err = lfs.PointerSmudgeToFile(cwdfilepath, pointer.Pointer, false, nil)
err = lfs.PointerSmudgeToFile(cwdfilepath, pointer.Pointer, false, manifest, nil)
if err != nil {
if errutil.IsDownloadDeclinedError(err) {
// acceptable error, data not local (fetch not run or include/exclude)

4
commands/command_smudge.go
View File

@ -8,6 +8,7 @@ import (
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/lfs"
"github.com/github/git-lfs/transfer"
"github.com/spf13/cobra"
)
@ -67,7 +68,8 @@ func smudgeCommand(cmd *cobra.Command, args []string) {
download = false
}
err = ptr.Smudge(os.Stdout, filename, download, cb)
manifest := transfer.ConfigureManifest(transfer.NewManifest(), cfg)
err = ptr.Smudge(os.Stdout, filename, download, manifest, cb)
if file != nil {
file.Close()
}

5
lfs/lfs.go
View File

@ -68,11 +68,12 @@ func ObjectExistsOfSize(oid string, size int64) bool {
}
func Environ() []string {
manifest := transfer.ConfigureManifest(transfer.NewManifest(), config.Config)
osEnviron := os.Environ()
env := make([]string, 0, len(osEnviron)+7)
dltransfers := transfer.GetDownloadAdapterNames()
dltransfers := manifest.GetDownloadAdapterNames()
sort.Strings(dltransfers)
ultransfers := transfer.GetUploadAdapterNames()
ultransfers := manifest.GetUploadAdapterNames()
sort.Strings(ultransfers)
env = append(env,

5
lfs/pointer.go
View File

@ -14,6 +14,7 @@ import (
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/progress"
"github.com/github/git-lfs/transfer"
)
var (
@ -60,8 +61,8 @@ func NewPointerExtension(name string, priority int, oid string) *PointerExtensio
return &PointerExtension{name, priority, oid, oidType}
}
func (p *Pointer) Smudge(writer io.Writer, workingfile string, download bool, cb progress.CopyCallback) error {
return PointerSmudge(writer, p, workingfile, download, cb)
func (p *Pointer) Smudge(writer io.Writer, workingfile string, download bool, manifest *transfer.Manifest, cb progress.CopyCallback) error {
return PointerSmudge(writer, p, workingfile, download, manifest, cb)
}
func (p *Pointer) Encode(writer io.Writer) (int, error) {

14
lfs/pointer_smudge.go
View File

@ -17,14 +17,14 @@ import (
"github.com/rubyist/tracerx"
)
func PointerSmudgeToFile(filename string, ptr *Pointer, download bool, cb progress.CopyCallback) error {
func PointerSmudgeToFile(filename string, ptr *Pointer, download bool, manifest *transfer.Manifest, cb progress.CopyCallback) error {
os.MkdirAll(filepath.Dir(filename), 0755)
file, err := os.Create(filename)
if err != nil {
return fmt.Errorf("Could not create working directory file: %v", err)
}
defer file.Close()
if err := PointerSmudge(file, ptr, filename, download, cb); err != nil {
if err := PointerSmudge(file, ptr, filename, download, manifest, cb); err != nil {
if errutil.IsDownloadDeclinedError(err) {
// write placeholder data instead
file.Seek(0, os.SEEK_SET)
@ -37,7 +37,7 @@ func PointerSmudgeToFile(filename string, ptr *Pointer, download bool, cb progre
return nil
}
func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download bool, cb progress.CopyCallback) error {
func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download bool, manifest *transfer.Manifest, cb progress.CopyCallback) error {
mediafile, err := LocalMediaPath(ptr.Oid)
if err != nil {
return err
@ -58,7 +58,7 @@ func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download
if statErr != nil || stat == nil {
if download {
err = downloadFile(writer, ptr, workingfile, mediafile, cb)
err = downloadFile(writer, ptr, workingfile, mediafile, manifest, cb)
} else {
return errutil.NewDownloadDeclinedError(nil)
}
@ -73,10 +73,10 @@ func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download
return nil
}
func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, cb progress.CopyCallback) error {
func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, manifest *transfer.Manifest, cb progress.CopyCallback) error {
fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, pb.FormatBytes(ptr.Size))
xfers := transfer.GetDownloadAdapterNames()
xfers := manifest.GetDownloadAdapterNames()
obj, adapterName, err := api.BatchOrLegacySingle(config.Config, &api.ObjectResource{Oid: ptr.Oid, Size: ptr.Size}, "download", xfers)
if err != nil {
return errutil.Errorf(err, "Error downloading %s: %s", filepath.Base(mediafile), err)
@ -86,7 +86,7 @@ func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string,
ptr.Size = obj.Size
}
adapter := transfer.NewDownloadAdapter(adapterName)
adapter := manifest.NewDownloadAdapter(adapterName)
var tcb transfer.TransferProgressCallback
if cb != nil {
tcb = func(name string, totalSize, readSoFar int64, readSinceLast int) error {

6
lfs/transfer_queue.go
View File

@ -53,6 +53,7 @@ type TransferQueue struct {
retrywait sync.WaitGroup
wait sync.WaitGroup // Incremented on Add(), decremented on transfer complete or skip
oldApiWorkers int // Number of non-batch API workers to spawn (deprecated)
manifest *transfer.Manifest
}
// newTransferQueue builds a TransferQueue, direction and underlying mechanism determined by adapter
@ -67,6 +68,7 @@ func newTransferQueue(files int, size int64, dryRun bool, dir transfer.Direction
oldApiWorkers: config.Config.ConcurrentTransfers(),
transferables: make(map[string]Transferable),
trMutex: &sync.Mutex{},
manifest: transfer.ConfigureManifest(transfer.NewManifest(), config.Config),
}
q.errorwait.Add(1)
@ -107,7 +109,7 @@ func (q *TransferQueue) useAdapter(name string) {
// changing adapter support in between batches
q.finishAdapter()
}
q.adapter = transfer.NewAdapterOrDefault(name, q.direction)
q.adapter = q.manifest.NewAdapterOrDefault(name, q.direction)
}
func (q *TransferQueue) finishAdapter() {
@ -330,7 +332,7 @@ func (q *TransferQueue) legacyFallback(failedBatch []interface{}) {
func (q *TransferQueue) batchApiRoutine() {
var startProgress sync.Once
transferAdapterNames := transfer.GetAdapterNames(q.direction)
transferAdapterNames := q.manifest.GetAdapterNames(q.direction)
for {
batch := q.batcher.Next()

8
transfer/basic_download.go
View File

@ -214,11 +214,10 @@ func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback
}
return tools.RenameFileCopyPermissions(dlfilename, t.Path)
}
func init() {
newfunc := func(name string, dir Direction) TransferAdapter {
func configureBasicDownloadAdapter(m *Manifest) {
m.RegisterNewTransferAdapterFunc(BasicAdapterName, Download, func(name string, dir Direction) TransferAdapter {
switch dir {
case Download:
bd := &basicDownloadAdapter{newAdapterBase(name, dir, nil)}
@ -229,6 +228,5 @@ func init() {
panic("Should never ask this func to upload")
}
return nil
}
RegisterNewTransferAdapterFunc(BasicAdapterName, Download, newfunc)
})
}

7
transfer/basic_upload.go
View File

@ -138,8 +138,8 @@ func newStartCallbackReader(r io.Reader, cb func(*startCallbackReader)) *startCa
return &startCallbackReader{r, cb, false}
}
func init() {
newfunc := func(name string, dir Direction) TransferAdapter {
func configureBasicUploadAdapter(m *Manifest) {
m.RegisterNewTransferAdapterFunc(BasicAdapterName, Upload, func(name string, dir Direction) TransferAdapter {
switch dir {
case Upload:
bu := &basicUploadAdapter{newAdapterBase(name, dir, nil)}
@ -150,6 +150,5 @@ func init() {
panic("Should never ask this func for basic download")
}
return nil
}
RegisterNewTransferAdapterFunc(BasicAdapterName, Upload, newfunc)
})
}

14
transfer/custom.go
View File

@ -347,9 +347,9 @@ func newCustomAdapter(name string, dir Direction, path, args string, concurrent
}
// Initialise custom adapters based on current config
func ConfigureCustomAdapters() {
func configureCustomAdapters(cfg *config.Configuration, m *Manifest) {
pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
for k, v := range config.Config.AllGitConfig() {
for k, v := range cfg.AllGitConfig() {
match := pathRegex.FindStringSubmatch(k)
if match == nil {
continue
@ -358,9 +358,9 @@ func ConfigureCustomAdapters() {
name := match[1]
path := v
// retrieve other values
args, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
concurrent := config.Config.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
direction, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
args, _ := cfg.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
concurrent := cfg.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
direction, _ := cfg.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
if len(direction) == 0 {
direction = "both"
} else {
@ -373,10 +373,10 @@ func ConfigureCustomAdapters() {
}
if direction == "download" || direction == "both" {
RegisterNewTransferAdapterFunc(name, Download, newfunc)
m.RegisterNewTransferAdapterFunc(name, Download, newfunc)
}
if direction == "upload" || direction == "both" {
RegisterNewTransferAdapterFunc(name, Upload, newfunc)
m.RegisterNewTransferAdapterFunc(name, Upload, newfunc)
}
}
}

125
transfer/custom_test.go
View File

@ -3,58 +3,19 @@ package transfer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/github/git-lfs/config"
"github.com/stretchr/testify/assert"
)
var (
savedDownloadAdapterFuncs map[string]NewTransferAdapterFunc
savedUploadAdapterFuncs map[string]NewTransferAdapterFunc
)
func copyFuncMap(to, from map[string]NewTransferAdapterFunc) {
for k, v := range from {
to[k] = v
}
}
func saveTransferSetupState() {
funcMutex.Lock()
defer funcMutex.Unlock()
savedDownloadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
copyFuncMap(savedDownloadAdapterFuncs, downloadAdapterFuncs)
savedUploadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
copyFuncMap(savedUploadAdapterFuncs, uploadAdapterFuncs)
}
func restoreTransferSetupState() {
funcMutex.Lock()
defer funcMutex.Unlock()
downloadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
copyFuncMap(downloadAdapterFuncs, savedDownloadAdapterFuncs)
uploadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
copyFuncMap(uploadAdapterFuncs, savedUploadAdapterFuncs)
}
func TestCustomTransferBasicConfig(t *testing.T) {
saveTransferSetupState()
defer func() {
config.Config.ResetConfig()
restoreTransferSetupState()
}()
path := "/path/to/binary"
config.Config.SetConfig("lfs.customtransfer.testsimple.path", path)
cfg := config.NewFrom(config.Values{
Git: map[string]string{"lfs.customtransfer.testsimple.path": path},
})
ConfigureCustomAdapters()
m := ConfigureManifest(NewManifest(), cfg)
u := NewUploadAdapter("testsimple")
u := m.NewUploadAdapter("testsimple")
assert.NotNil(t, u, "Upload adapter should be present")
cu, _ := u.(*customAdapter)
assert.NotNil(t, cu, "Upload adapter should be customAdapter")
@ -62,7 +23,7 @@ func TestCustomTransferBasicConfig(t *testing.T) {
assert.Equal(t, cu.args, "", "args should be blank")
assert.Equal(t, cu.concurrent, true, "concurrent should be defaulted")
d := NewDownloadAdapter("testsimple")
d := m.NewDownloadAdapter("testsimple")
assert.NotNil(t, d, "Download adapter should be present")
cd, _ := u.(*customAdapter)
assert.NotNil(t, cd, "Download adapter should be customAdapter")
@ -72,27 +33,25 @@ func TestCustomTransferBasicConfig(t *testing.T) {
}
func TestCustomTransferDownloadConfig(t *testing.T) {
saveTransferSetupState()
defer func() {
config.Config.ResetConfig()
restoreTransferSetupState()
}()
path := "/path/to/binary"
args := "-c 1 --whatever"
config.Config.SetConfig("lfs.customtransfer.testdownload.path", path)
config.Config.SetConfig("lfs.customtransfer.testdownload.args", args)
config.Config.SetConfig("lfs.customtransfer.testdownload.concurrent", "false")
config.Config.SetConfig("lfs.customtransfer.testdownload.direction", "download")
cfg := config.NewFrom(config.Values{
Git: map[string]string{
"lfs.customtransfer.testdownload.path": path,
"lfs.customtransfer.testdownload.args": args,
"lfs.customtransfer.testdownload.concurrent": "false",
"lfs.customtransfer.testdownload.direction": "download",
},
})
ConfigureCustomAdapters()
m := ConfigureManifest(NewManifest(), cfg)
u := NewUploadAdapter("testdownload")
u := m.NewUploadAdapter("testdownload")
assert.NotNil(t, u, "Upload adapter should always be created")
cu, _ := u.(*customAdapter)
assert.Nil(t, cu, "Upload adapter should NOT be custom (default to basic)")
d := NewDownloadAdapter("testdownload")
d := m.NewDownloadAdapter("testdownload")
assert.NotNil(t, d, "Download adapter should be present")
cd, _ := d.(*customAdapter)
assert.NotNil(t, cd, "Download adapter should be customAdapter")
@ -102,27 +61,25 @@ func TestCustomTransferDownloadConfig(t *testing.T) {
}
func TestCustomTransferUploadConfig(t *testing.T) {
saveTransferSetupState()
defer func() {
config.Config.ResetConfig()
restoreTransferSetupState()
}()
path := "/path/to/binary"
args := "-c 1 --whatever"
config.Config.SetConfig("lfs.customtransfer.testupload.path", path)
config.Config.SetConfig("lfs.customtransfer.testupload.args", args)
config.Config.SetConfig("lfs.customtransfer.testupload.concurrent", "false")
config.Config.SetConfig("lfs.customtransfer.testupload.direction", "upload")
cfg := config.NewFrom(config.Values{
Git: map[string]string{
"lfs.customtransfer.testupload.path": path,
"lfs.customtransfer.testupload.args": args,
"lfs.customtransfer.testupload.concurrent": "false",
"lfs.customtransfer.testupload.direction": "upload",
},
})
ConfigureCustomAdapters()
m := ConfigureManifest(NewManifest(), cfg)
d := NewDownloadAdapter("testupload")
d := m.NewDownloadAdapter("testupload")
assert.NotNil(t, d, "Download adapter should always be created")
cd, _ := d.(*customAdapter)
assert.Nil(t, cd, "Download adapter should NOT be custom (default to basic)")
u := NewUploadAdapter("testupload")
u := m.NewUploadAdapter("testupload")
assert.NotNil(t, u, "Upload adapter should be present")
cu, _ := u.(*customAdapter)
assert.NotNil(t, cu, "Upload adapter should be customAdapter")
@ -132,22 +89,20 @@ func TestCustomTransferUploadConfig(t *testing.T) {
}
func TestCustomTransferBothConfig(t *testing.T) {
saveTransferSetupState()
defer func() {
config.Config.ResetConfig()
restoreTransferSetupState()
}()
path := "/path/to/binary"
args := "-c 1 --whatever --yeah"
config.Config.SetConfig("lfs.customtransfer.testboth.path", path)
config.Config.SetConfig("lfs.customtransfer.testboth.args", args)
config.Config.SetConfig("lfs.customtransfer.testboth.concurrent", "yes")
config.Config.SetConfig("lfs.customtransfer.testboth.direction", "both")
cfg := config.NewFrom(config.Values{
Git: map[string]string{
"lfs.customtransfer.testboth.path": path,
"lfs.customtransfer.testboth.args": args,
"lfs.customtransfer.testboth.concurrent": "yes",
"lfs.customtransfer.testboth.direction": "both",
},
})
ConfigureCustomAdapters()
m := ConfigureManifest(NewManifest(), cfg)
d := NewDownloadAdapter("testboth")
d := m.NewDownloadAdapter("testboth")
assert.NotNil(t, d, "Download adapter should be present")
cd, _ := d.(*customAdapter)
assert.NotNil(t, cd, "Download adapter should be customAdapter")
@ -155,7 +110,7 @@ func TestCustomTransferBothConfig(t *testing.T) {
assert.Equal(t, cd.args, args, "args should be correct")
assert.Equal(t, cd.concurrent, true, "concurrent should be set")
u := NewUploadAdapter("testboth")
u := m.NewUploadAdapter("testboth")
assert.NotNil(t, u, "Upload adapter should be present")
cu, _ := u.(*customAdapter)
assert.NotNil(t, cu, "Upload adapter should be customAdapter")

128
transfer/manifest.go Normal file
View File

@ -0,0 +1,128 @@
package transfer
import (
"sync"
"github.com/github/git-lfs/config"
"github.com/rubyist/tracerx"
)
type Manifest struct {
basicTransfersOnly bool
downloadAdapterFuncs map[string]NewTransferAdapterFunc
uploadAdapterFuncs map[string]NewTransferAdapterFunc
mu sync.Mutex
}
func NewManifest() *Manifest {
return &Manifest{
downloadAdapterFuncs: make(map[string]NewTransferAdapterFunc),
uploadAdapterFuncs: make(map[string]NewTransferAdapterFunc),
}
}
func ConfigureManifest(m *Manifest, cfg *config.Configuration) *Manifest {
m.basicTransfersOnly = cfg.BasicTransfersOnly()
configureBasicDownloadAdapter(m)
configureBasicUploadAdapter(m)
if cfg.TusTransfersAllowed() {
configureTusAdapter(m)
}
configureCustomAdapters(cfg, m)
return m
}
// GetAdapterNames returns a list of the names of adapters available to be created
func (m *Manifest) GetAdapterNames(dir Direction) []string {
switch dir {
case Upload:
return m.GetUploadAdapterNames()
case Download:
return m.GetDownloadAdapterNames()
}
return nil
}
// GetDownloadAdapterNames returns a list of the names of download adapters available to be created
func (m *Manifest) GetDownloadAdapterNames() []string {
return m.getAdapterNames(m.downloadAdapterFuncs)
}
// GetUploadAdapterNames returns a list of the names of upload adapters available to be created
func (m *Manifest) GetUploadAdapterNames() []string {
return m.getAdapterNames(m.uploadAdapterFuncs)
}
// getAdapterNames returns a list of the names of adapters available to be created
func (m *Manifest) getAdapterNames(adapters map[string]NewTransferAdapterFunc) []string {
if m.basicTransfersOnly {
return []string{BasicAdapterName}
}
m.mu.Lock()
defer m.mu.Unlock()
ret := make([]string, 0, len(adapters))
for n, _ := range adapters {
ret = append(ret, n)
}
return ret
}
// RegisterNewTransferAdapterFunc registers a new function for creating upload
// or download adapters. If a function with that name & direction is already
// registered, it is overridden
func (m *Manifest) RegisterNewTransferAdapterFunc(name string, dir Direction, f NewTransferAdapterFunc) {
m.mu.Lock()
defer m.mu.Unlock()
switch dir {
case Upload:
m.uploadAdapterFuncs[name] = f
case Download:
m.downloadAdapterFuncs[name] = f
}
}
// Create a new adapter by name and direction; default to BasicAdapterName if doesn't exist
func (m *Manifest) NewAdapterOrDefault(name string, dir Direction) TransferAdapter {
if len(name) == 0 {
name = BasicAdapterName
}
a := m.NewAdapter(name, dir)
if a == nil {
tracerx.Printf("Defaulting to basic transfer adapter since %q did not exist", name)
a = m.NewAdapter(BasicAdapterName, dir)
}
return a
}
// Create a new adapter by name and direction, or nil if doesn't exist
func (m *Manifest) NewAdapter(name string, dir Direction) TransferAdapter {
m.mu.Lock()
defer m.mu.Unlock()
switch dir {
case Upload:
if u, ok := m.uploadAdapterFuncs[name]; ok {
return u(name, dir)
}
case Download:
if d, ok := m.downloadAdapterFuncs[name]; ok {
return d(name, dir)
}
}
return nil
}
// Create a new download adapter by name, or BasicAdapterName if doesn't exist
func (m *Manifest) NewDownloadAdapter(name string) TransferAdapter {
return m.NewAdapterOrDefault(name, Download)
}
// Create a new upload adapter by name, or BasicAdapterName if doesn't exist
func (m *Manifest) NewUploadAdapter(name string) TransferAdapter {
return m.NewAdapterOrDefault(name, Upload)
}

142
transfer/transfer.go
View File

@ -2,14 +2,7 @@
// NOTE: Subject to change, do not rely on this package from outside git-lfs source
package transfer
import (
"sync"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/api"
"github.com/rubyist/tracerx"
)
import "github.com/github/git-lfs/api"
type Direction int
@ -20,16 +13,10 @@ const (
// NewTransferAdapterFunc creates new instances of TransferAdapter. Code that wishes
// to provide new TransferAdapter instances should pass an implementation of this
// function to RegisterNewTransferAdapterFunc
// function to RegisterNewTransferAdapterFunc() on a *Manifest.
// name and dir are to provide context if one func implements many instances
type NewTransferAdapterFunc func(name string, dir Direction) TransferAdapter
var (
funcMutex sync.Mutex
downloadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
uploadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
)
type TransferProgressCallback func(name string, totalSize, readSoFar int64, readSinceLast int) error
// TransferAdapter is implemented by types which can upload and/or download LFS
@ -92,128 +79,3 @@ type TransferResult struct {
// This will be non-nil if there was an error transferring this item
Error error
}
// GetAdapterNames returns a list of the names of adapters available to be created
func GetAdapterNames(dir Direction) []string {
switch dir {
case Upload:
return GetUploadAdapterNames()
case Download:
return GetDownloadAdapterNames()
}
return nil
}
// GetDownloadAdapterNames returns a list of the names of download adapters available to be created
func GetDownloadAdapterNames() []string {
if config.Config.BasicTransfersOnly() {
return []string{BasicAdapterName}
}
initCoreAdaptersIfRequired()
funcMutex.Lock()
defer funcMutex.Unlock()
ret := make([]string, 0, len(downloadAdapterFuncs))
for n, _ := range downloadAdapterFuncs {
ret = append(ret, n)
}
return ret
}
// GetUploadAdapterNames returns a list of the names of upload adapters available to be created
func GetUploadAdapterNames() []string {
if config.Config.BasicTransfersOnly() {
return []string{BasicAdapterName}
}
initCoreAdaptersIfRequired()
funcMutex.Lock()
defer funcMutex.Unlock()
ret := make([]string, 0, len(uploadAdapterFuncs))
for n, _ := range uploadAdapterFuncs {
ret = append(ret, n)
}
return ret
}
// RegisterNewTransferAdapterFunc registers a new function for creating upload
// or download adapters. If a function with that name & direction is already
// registered, it is overridden
func RegisterNewTransferAdapterFunc(name string, dir Direction, f NewTransferAdapterFunc) {
funcMutex.Lock()
defer funcMutex.Unlock()
switch dir {
case Upload:
uploadAdapterFuncs[name] = f
case Download:
downloadAdapterFuncs[name] = f
}
}
// Create a new adapter by name and direction; default to BasicAdapterName if doesn't exist
func NewAdapterOrDefault(name string, dir Direction) TransferAdapter {
if len(name) == 0 {
name = BasicAdapterName
}
a := NewAdapter(name, dir)
if a == nil {
tracerx.Printf("Defaulting to basic transfer adapter since %q did not exist", name)
a = NewAdapter(BasicAdapterName, dir)
}
return a
}
// Create a new adapter by name and direction, or nil if doesn't exist
func NewAdapter(name string, dir Direction) TransferAdapter {
initCoreAdaptersIfRequired()
funcMutex.Lock()
defer funcMutex.Unlock()
switch dir {
case Upload:
if u, ok := uploadAdapterFuncs[name]; ok {
return u(name, dir)
}
case Download:
if d, ok := downloadAdapterFuncs[name]; ok {
return d(name, dir)
}
}
return nil
}
// Create a new download adapter by name, or BasicAdapterName if doesn't exist
func NewDownloadAdapter(name string) TransferAdapter {
return NewAdapterOrDefault(name, Download)
}
// Create a new upload adapter by name, or BasicAdapterName if doesn't exist
func NewUploadAdapter(name string) TransferAdapter {
return NewAdapterOrDefault(name, Upload)
}
var initCoreOnce sync.Once
func initCoreAdaptersIfRequired() {
// It's important to late-init custom adapters because they rely on Config
// And if we cause Config to load too early it causes issues
// That's why this isn't in an init() block
initCoreOnce.Do(func() {
ConfigureCustomAdapters()
// tus.io upload adapter is still experimental, requires
// `lfs.tustransfers=true` to activate.
if !config.Config.TusTransfersAllowed() {
delete(uploadAdapterFuncs, TusAdapterName)
}
})
}

74
transfer/transfer_test.go
View File

@ -16,47 +16,55 @@ type testAdapter struct {
func (a *testAdapter) Name() string {
return a.name
}
func (a *testAdapter) Direction() Direction {
return a.dir
}
func (a *testAdapter) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
return nil
}
func (a *testAdapter) Add(t *Transfer) {
}
func (a *testAdapter) End() {
}
func (a *testAdapter) ClearTempStorage() error {
return nil
}
func newTestAdapter(name string, dir Direction) TransferAdapter {
return &testAdapter{name, dir}
}
func newRenamedTestAdapter(name string, dir Direction) TransferAdapter {
return &testAdapter{"RENAMED", dir}
}
func resetAdapters() {
uploadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
downloadAdapterFuncs = make(map[string]NewTransferAdapterFunc)
}
func testBasicAdapterExists(t *testing.T) {
cfg := config.New()
m := ConfigureManifest(NewManifest(), cfg)
assert := assert.New(t)
dls := GetDownloadAdapterNames()
dls := m.GetDownloadAdapterNames()
if assert.NotNil(dls) {
assert.Equal([]string{"basic"}, dls)
}
uls := GetUploadAdapterNames()
uls := m.GetUploadAdapterNames()
if assert.NotNil(uls) {
assert.Equal([]string{"basic"}, uls)
}
da := NewDownloadAdapter("basic")
da := m.NewDownloadAdapter("basic")
if assert.NotNil(da) {
assert.Equal("basic", da.Name())
assert.Equal(Download, da.Direction())
}
ua := NewUploadAdapter("basic")
ua := m.NewUploadAdapter("basic")
if assert.NotNil(ua) {
assert.Equal("basic", ua.Name())
assert.Equal(Upload, ua.Direction())
@ -64,61 +72,69 @@ func testBasicAdapterExists(t *testing.T) {
}
func testAdapterRegAndOverride(t *testing.T) {
cfg := config.New()
m := ConfigureManifest(NewManifest(), cfg)
assert := assert.New(t)
assert.Nil(NewDownloadAdapter("test"))
assert.Nil(NewUploadAdapter("test"))
assert.Nil(m.NewDownloadAdapter("test"))
assert.Nil(m.NewUploadAdapter("test"))
RegisterNewTransferAdapterFunc("test", Upload, newTestAdapter)
assert.Nil(NewDownloadAdapter("test"))
assert.NotNil(NewUploadAdapter("test"))
m.RegisterNewTransferAdapterFunc("test", Upload, newTestAdapter)
assert.Nil(m.NewDownloadAdapter("test"))
assert.NotNil(m.NewUploadAdapter("test"))
RegisterNewTransferAdapterFunc("test", Download, newTestAdapter)
da := NewDownloadAdapter("test")
m.RegisterNewTransferAdapterFunc("test", Download, newTestAdapter)
da := m.NewDownloadAdapter("test")
if assert.NotNil(da) {
assert.Equal("test", da.Name())
assert.Equal(Download, da.Direction())
}
ua := NewUploadAdapter("test")
ua := m.NewUploadAdapter("test")
if assert.NotNil(ua) {
assert.Equal("test", ua.Name())
assert.Equal(Upload, ua.Direction())
}
// Test override
RegisterNewTransferAdapterFunc("test", Upload, newRenamedTestAdapter)
ua = NewUploadAdapter("test")
m.RegisterNewTransferAdapterFunc("test", Upload, newRenamedTestAdapter)
ua = m.NewUploadAdapter("test")
if assert.NotNil(ua) {
assert.Equal("RENAMED", ua.Name())
assert.Equal(Upload, ua.Direction())
}
da = NewDownloadAdapter("test")
da = m.NewDownloadAdapter("test")
if assert.NotNil(da) {
assert.Equal("test", da.Name())
assert.Equal(Download, da.Direction())
}
RegisterNewTransferAdapterFunc("test", Download, newRenamedTestAdapter)
da = NewDownloadAdapter("test")
m.RegisterNewTransferAdapterFunc("test", Download, newRenamedTestAdapter)
da = m.NewDownloadAdapter("test")
if assert.NotNil(da) {
assert.Equal("RENAMED", da.Name())
assert.Equal(Download, da.Direction())
}
}
func testAdapterRegButBasicOnly(t *testing.T) {
cfg := config.NewFrom(config.Values{
Git: map[string]string{"lfs.basictransfersonly": "yes"},
})
m := ConfigureManifest(NewManifest(), cfg)
assert := assert.New(t)
config.Config.SetConfig("lfs.basictransfersonly", "yes")
RegisterNewTransferAdapterFunc("test", Upload, newTestAdapter)
RegisterNewTransferAdapterFunc("test", Download, newTestAdapter)
m.RegisterNewTransferAdapterFunc("test", Upload, newTestAdapter)
m.RegisterNewTransferAdapterFunc("test", Download, newTestAdapter)
// Will still be created if we ask for them
assert.NotNil(NewUploadAdapter("test"))
assert.NotNil(NewDownloadAdapter("test"))
assert.NotNil(m.NewUploadAdapter("test"))
assert.NotNil(m.NewDownloadAdapter("test"))
// But list will exclude
ld := GetDownloadAdapterNames()
ld := m.GetDownloadAdapterNames()
assert.Equal([]string{BasicAdapterName}, ld)
lu := GetUploadAdapterNames()
lu := m.GetUploadAdapterNames()
assert.Equal([]string{BasicAdapterName}, lu)
}

7
transfer/tus_upload.go
View File

@ -156,8 +156,8 @@ func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferP
return api.VerifyUpload(config.Config, t.Object)
}
func init() {
newfunc := func(name string, dir Direction) TransferAdapter {
func configureTusAdapter(m *Manifest) {
m.RegisterNewTransferAdapterFunc(TusAdapterName, Upload, func(name string, dir Direction) TransferAdapter {
switch dir {
case Upload:
bu := &tusUploadAdapter{newAdapterBase(name, dir, nil)}
@ -168,6 +168,5 @@ func init() {
panic("Should never ask tus.io to download")
}
return nil
}
RegisterNewTransferAdapterFunc(TusAdapterName, Upload, newfunc)
})
}