2015-07-23 16:53:58 +00:00
|
|
|
package commands
|
|
|
|
|
|
|
|
import (
|
2015-10-06 16:35:58 +00:00
|
|
|
"fmt"
|
2016-12-14 18:36:55 +00:00
|
|
|
"sync"
|
2016-12-14 22:17:24 +00:00
|
|
|
"time"
|
2015-10-06 16:35:58 +00:00
|
|
|
|
2016-11-21 22:14:33 +00:00
|
|
|
"github.com/git-lfs/git-lfs/filepathfilter"
|
2016-11-15 17:01:18 +00:00
|
|
|
"github.com/git-lfs/git-lfs/git"
|
2016-12-14 18:36:55 +00:00
|
|
|
"github.com/git-lfs/git-lfs/lfs"
|
2016-12-14 22:17:24 +00:00
|
|
|
"github.com/git-lfs/git-lfs/progress"
|
2016-12-16 17:35:38 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tq"
|
2016-12-14 22:17:24 +00:00
|
|
|
"github.com/rubyist/tracerx"
|
2016-05-23 18:02:27 +00:00
|
|
|
"github.com/spf13/cobra"
|
2015-07-23 16:53:58 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func pullCommand(cmd *cobra.Command, args []string) {
|
2016-08-17 22:13:36 +00:00
|
|
|
requireGitVersion()
|
2015-09-08 16:23:27 +00:00
|
|
|
requireInRepo()
|
2015-07-23 16:53:58 +00:00
|
|
|
|
2017-01-04 15:56:09 +00:00
|
|
|
var remote string
|
2015-08-13 16:22:29 +00:00
|
|
|
if len(args) > 0 {
|
|
|
|
// Remote is first arg
|
2015-10-06 16:35:58 +00:00
|
|
|
if err := git.ValidateRemote(args[0]); err != nil {
|
|
|
|
Panic(err, fmt.Sprintf("Invalid remote name '%v'", args[0]))
|
|
|
|
}
|
2017-01-04 15:56:09 +00:00
|
|
|
remote = args[0]
|
2015-08-13 16:22:29 +00:00
|
|
|
} else {
|
2015-10-06 16:27:36 +00:00
|
|
|
// Actively find the default remote, don't just assume origin
|
|
|
|
defaultRemote, err := git.DefaultRemote()
|
|
|
|
if err != nil {
|
|
|
|
Panic(err, "No default remote")
|
|
|
|
}
|
2017-01-04 15:56:09 +00:00
|
|
|
remote = defaultRemote
|
2015-08-13 16:22:29 +00:00
|
|
|
}
|
|
|
|
|
2016-08-02 16:37:16 +00:00
|
|
|
includeArg, excludeArg := getIncludeExcludeArgs(cmd)
|
2016-11-21 22:14:33 +00:00
|
|
|
filter := buildFilepathFilter(cfg, includeArg, excludeArg)
|
2017-01-04 15:56:09 +00:00
|
|
|
pull(remote, filter)
|
2016-02-05 17:45:12 +00:00
|
|
|
}
|
|
|
|
|
2017-01-04 15:56:09 +00:00
|
|
|
func pull(remote string, filter *filepathfilter.Filter) {
|
|
|
|
cfg.CurrentRemote = remote
|
2015-07-23 16:53:58 +00:00
|
|
|
ref, err := git.CurrentRef()
|
|
|
|
if err != nil {
|
|
|
|
Panic(err, "Could not pull")
|
|
|
|
}
|
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
pointers := newPointerMap()
|
|
|
|
meter := progress.NewMeter(progress.WithOSEnv(cfg.Os))
|
2017-01-04 21:46:30 +00:00
|
|
|
singleCheckout := newSingleCheckout()
|
|
|
|
q := newDownloadQueue(singleCheckout.manifest, remote, tq.WithProgress(meter))
|
2016-12-14 22:17:24 +00:00
|
|
|
gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
|
2016-12-14 18:36:55 +00:00
|
|
|
if err != nil {
|
2017-04-26 15:11:48 +00:00
|
|
|
LoggedError(err, "Scanner error: %s", err)
|
2016-12-14 18:36:55 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
if pointers.Seen(p) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// no need to download objects that exist locally already
|
|
|
|
lfs.LinkOrCopyFromReference(p.Oid, p.Size)
|
|
|
|
if lfs.ObjectExistsOfSize(p.Oid, p.Size) {
|
|
|
|
singleCheckout.Run(p)
|
|
|
|
return
|
|
|
|
}
|
2016-12-14 18:36:55 +00:00
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
meter.Add(p.Size)
|
|
|
|
meter.StartTransfer(p.Name)
|
|
|
|
tracerx.Printf("fetch %v [%v]", p.Name, p.Oid)
|
|
|
|
pointers.Add(p)
|
2016-12-16 17:35:38 +00:00
|
|
|
q.Add(downloadTransfer(p))
|
2016-12-14 22:17:24 +00:00
|
|
|
})
|
2016-12-14 18:36:55 +00:00
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
gitscanner.Filter = filter
|
2016-12-14 18:36:55 +00:00
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
dlwatch := q.Watch()
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
wg.Add(1)
|
2016-12-14 18:36:55 +00:00
|
|
|
|
|
|
|
go func() {
|
2017-08-02 19:17:39 +00:00
|
|
|
for t := range dlwatch {
|
|
|
|
for _, p := range pointers.All(t.Oid) {
|
2016-12-14 22:17:24 +00:00
|
|
|
singleCheckout.Run(p)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
wg.Done()
|
2016-12-14 18:36:55 +00:00
|
|
|
}()
|
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
processQueue := time.Now()
|
|
|
|
if err := gitscanner.ScanTree(ref.Sha); err != nil {
|
2017-04-27 20:35:57 +00:00
|
|
|
singleCheckout.Close()
|
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
ExitWithError(err)
|
2016-12-14 18:36:55 +00:00
|
|
|
}
|
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
meter.Start()
|
|
|
|
gitscanner.Close()
|
|
|
|
q.Wait()
|
|
|
|
wg.Wait()
|
|
|
|
tracerx.PerformanceSince("process queue", processQueue)
|
2016-12-14 19:01:53 +00:00
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
singleCheckout.Close()
|
2016-12-14 19:01:53 +00:00
|
|
|
|
2017-05-19 01:04:59 +00:00
|
|
|
success := true
|
2016-12-14 22:17:24 +00:00
|
|
|
for _, err := range q.Errors() {
|
2017-05-19 01:04:59 +00:00
|
|
|
success = false
|
2016-12-14 22:17:24 +00:00
|
|
|
FullError(err)
|
|
|
|
}
|
2017-05-19 01:04:59 +00:00
|
|
|
|
|
|
|
if !success {
|
2017-05-19 16:35:49 +00:00
|
|
|
c := getAPIClient()
|
|
|
|
e := c.Endpoints.Endpoint("download", remote)
|
2017-06-14 09:24:30 +00:00
|
|
|
Exit("error: failed to fetch some objects from '%s'", e.Url)
|
2017-05-19 01:04:59 +00:00
|
|
|
}
|
2016-12-14 22:17:24 +00:00
|
|
|
}
|
2016-12-14 19:01:53 +00:00
|
|
|
|
2016-12-14 22:29:03 +00:00
|
|
|
// tracks LFS objects being downloaded, according to their unique OIDs.
|
2016-12-14 22:17:24 +00:00
|
|
|
type pointerMap struct {
|
|
|
|
pointers map[string][]*lfs.WrappedPointer
|
|
|
|
mu sync.Mutex
|
|
|
|
}
|
2016-12-14 19:01:53 +00:00
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
func newPointerMap() *pointerMap {
|
|
|
|
return &pointerMap{pointers: make(map[string][]*lfs.WrappedPointer)}
|
|
|
|
}
|
2016-12-14 19:01:53 +00:00
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
func (m *pointerMap) Seen(p *lfs.WrappedPointer) bool {
|
|
|
|
m.mu.Lock()
|
2016-12-14 22:48:59 +00:00
|
|
|
defer m.mu.Unlock()
|
|
|
|
if existing, ok := m.pointers[p.Oid]; ok {
|
2016-12-14 22:17:24 +00:00
|
|
|
m.pointers[p.Oid] = append(existing, p)
|
2016-12-14 22:48:59 +00:00
|
|
|
return true
|
2016-12-14 19:01:53 +00:00
|
|
|
}
|
2016-12-14 22:48:59 +00:00
|
|
|
return false
|
2016-12-14 22:17:24 +00:00
|
|
|
}
|
2016-12-14 19:01:53 +00:00
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
func (m *pointerMap) Add(p *lfs.WrappedPointer) {
|
|
|
|
m.mu.Lock()
|
|
|
|
defer m.mu.Unlock()
|
|
|
|
m.pointers[p.Oid] = append(m.pointers[p.Oid], p)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *pointerMap) All(oid string) []*lfs.WrappedPointer {
|
|
|
|
m.mu.Lock()
|
2016-12-14 22:48:59 +00:00
|
|
|
defer m.mu.Unlock()
|
2016-12-14 22:17:24 +00:00
|
|
|
pointers := m.pointers[oid]
|
|
|
|
delete(m.pointers, oid)
|
|
|
|
return pointers
|
2016-12-14 19:01:53 +00:00
|
|
|
}
|
|
|
|
|
2015-07-23 16:53:58 +00:00
|
|
|
func init() {
|
2016-09-01 16:09:38 +00:00
|
|
|
RegisterCommand("pull", pullCommand, func(cmd *cobra.Command) {
|
2016-08-10 15:33:25 +00:00
|
|
|
cmd.Flags().StringVarP(&includeArg, "include", "I", "", "Include a list of paths")
|
|
|
|
cmd.Flags().StringVarP(&excludeArg, "exclude", "X", "", "Exclude a list of paths")
|
|
|
|
})
|
2015-07-23 16:53:58 +00:00
|
|
|
}
|