2015-05-14 16:33:29 +00:00
|
|
|
package commands
|
|
|
|
|
|
|
|
import (
|
2015-09-03 16:23:30 +00:00
|
|
|
"fmt"
|
2015-07-28 13:50:19 +00:00
|
|
|
"time"
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
"github.com/github/git-lfs/config"
|
2015-05-14 16:33:29 +00:00
|
|
|
"github.com/github/git-lfs/git"
|
|
|
|
"github.com/github/git-lfs/lfs"
|
2016-05-06 13:46:13 +00:00
|
|
|
"github.com/github/git-lfs/progress"
|
2016-05-23 18:02:27 +00:00
|
|
|
"github.com/rubyist/tracerx"
|
|
|
|
"github.com/spf13/cobra"
|
2015-05-14 16:33:29 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
2015-05-29 15:28:10 +00:00
|
|
|
fetchCmd = &cobra.Command{
|
2015-09-17 22:08:28 +00:00
|
|
|
Use: "fetch",
|
|
|
|
Run: fetchCommand,
|
2015-05-14 16:33:29 +00:00
|
|
|
}
|
2015-08-07 17:18:39 +00:00
|
|
|
fetchIncludeArg string
|
|
|
|
fetchExcludeArg string
|
2015-08-20 14:36:10 +00:00
|
|
|
fetchRecentArg bool
|
2015-09-03 11:29:00 +00:00
|
|
|
fetchAllArg bool
|
2015-10-15 15:35:26 +00:00
|
|
|
fetchPruneArg bool
|
2015-05-14 16:33:29 +00:00
|
|
|
)
|
|
|
|
|
2015-05-29 15:28:10 +00:00
|
|
|
func fetchCommand(cmd *cobra.Command, args []string) {
|
2015-09-08 16:23:27 +00:00
|
|
|
requireInRepo()
|
2015-05-14 16:33:29 +00:00
|
|
|
|
2015-09-08 16:23:27 +00:00
|
|
|
var refs []*git.Ref
|
2015-09-08 15:29:53 +00:00
|
|
|
|
2015-07-29 15:34:15 +00:00
|
|
|
if len(args) > 0 {
|
2015-08-13 16:22:29 +00:00
|
|
|
// Remote is first arg
|
2015-10-06 16:35:58 +00:00
|
|
|
if err := git.ValidateRemote(args[0]); err != nil {
|
2015-10-07 08:13:20 +00:00
|
|
|
Exit("Invalid remote name %q", args[0])
|
2015-10-06 16:35:58 +00:00
|
|
|
}
|
2016-05-13 16:38:06 +00:00
|
|
|
config.Config.CurrentRemote = args[0]
|
2015-08-13 16:22:29 +00:00
|
|
|
} else {
|
2015-10-06 16:27:36 +00:00
|
|
|
// Actively find the default remote, don't just assume origin
|
|
|
|
defaultRemote, err := git.DefaultRemote()
|
|
|
|
if err != nil {
|
2015-10-07 08:13:20 +00:00
|
|
|
Exit("No default remote")
|
2015-10-06 16:27:36 +00:00
|
|
|
}
|
2016-05-13 16:38:06 +00:00
|
|
|
config.Config.CurrentRemote = defaultRemote
|
2015-08-13 16:22:29 +00:00
|
|
|
}
|
2015-09-08 15:29:53 +00:00
|
|
|
|
2015-08-13 16:22:29 +00:00
|
|
|
if len(args) > 1 {
|
2016-04-06 16:29:59 +00:00
|
|
|
resolvedrefs, err := git.ResolveRefs(args[1:])
|
|
|
|
if err != nil {
|
|
|
|
Panic(err, "Invalid ref argument: %v", args[1:])
|
2015-08-21 14:19:16 +00:00
|
|
|
}
|
2016-04-06 16:29:59 +00:00
|
|
|
refs = resolvedrefs
|
2015-05-14 16:33:29 +00:00
|
|
|
} else {
|
2015-07-29 15:34:15 +00:00
|
|
|
ref, err := git.CurrentRef()
|
2015-05-14 16:33:29 +00:00
|
|
|
if err != nil {
|
2015-05-29 15:28:10 +00:00
|
|
|
Panic(err, "Could not fetch")
|
2015-05-14 16:33:29 +00:00
|
|
|
}
|
2015-08-21 14:19:16 +00:00
|
|
|
refs = []*git.Ref{ref}
|
2015-07-29 15:34:15 +00:00
|
|
|
}
|
|
|
|
|
2015-10-13 14:29:25 +00:00
|
|
|
success := true
|
2015-09-03 11:29:00 +00:00
|
|
|
if fetchAllArg {
|
2015-09-03 16:36:27 +00:00
|
|
|
if fetchRecentArg || len(args) > 1 {
|
2015-09-03 11:29:00 +00:00
|
|
|
Exit("Cannot combine --all with ref arguments or --recent")
|
|
|
|
}
|
|
|
|
if fetchIncludeArg != "" || fetchExcludeArg != "" {
|
|
|
|
Exit("Cannot combine --all with --include or --exclude")
|
|
|
|
}
|
2016-05-27 19:12:12 +00:00
|
|
|
if len(config.Config.GlobalFetchIncludePaths()) > 0 || len(config.Config.GlobalFetchExcludePaths()) > 0 {
|
2015-09-03 11:29:00 +00:00
|
|
|
Print("Ignoring global include / exclude paths to fulfil --all")
|
|
|
|
}
|
2015-10-13 14:29:25 +00:00
|
|
|
success = fetchAll()
|
2015-05-14 16:33:29 +00:00
|
|
|
|
2015-09-03 11:29:00 +00:00
|
|
|
} else { // !all
|
2016-05-27 19:12:12 +00:00
|
|
|
includePaths, excludePaths := config.Config.FetchIncludeExcludePaths(fetchIncludeArg, fetchExcludeArg)
|
2015-09-03 11:29:00 +00:00
|
|
|
|
|
|
|
// Fetch refs sequentially per arg order; duplicates in later refs will be ignored
|
|
|
|
for _, ref := range refs {
|
|
|
|
Print("Fetching %v", ref.Name)
|
2015-10-13 14:29:25 +00:00
|
|
|
s := fetchRef(ref.Sha, includePaths, excludePaths)
|
|
|
|
success = success && s
|
2015-09-03 11:29:00 +00:00
|
|
|
}
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
if fetchRecentArg || config.Config.FetchPruneConfig().FetchRecentAlways {
|
2015-10-13 14:29:25 +00:00
|
|
|
s := fetchRecent(refs, includePaths, excludePaths)
|
|
|
|
success = success && s
|
2015-09-03 11:29:00 +00:00
|
|
|
}
|
|
|
|
}
|
2015-10-13 14:29:25 +00:00
|
|
|
|
2015-10-15 15:35:26 +00:00
|
|
|
if fetchPruneArg {
|
2016-05-13 16:38:06 +00:00
|
|
|
verify := config.Config.FetchPruneConfig().PruneVerifyRemoteAlways
|
2015-10-15 15:35:26 +00:00
|
|
|
// no dry-run or verbose options in fetch, assume false
|
|
|
|
prune(verify, false, false)
|
|
|
|
}
|
2015-11-09 17:46:29 +00:00
|
|
|
|
2015-10-13 14:29:25 +00:00
|
|
|
if !success {
|
|
|
|
Exit("Warning: errors occurred")
|
|
|
|
}
|
2015-07-23 16:53:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
2015-08-07 17:18:39 +00:00
|
|
|
fetchCmd.Flags().StringVarP(&fetchIncludeArg, "include", "I", "", "Include a list of paths")
|
|
|
|
fetchCmd.Flags().StringVarP(&fetchExcludeArg, "exclude", "X", "", "Exclude a list of paths")
|
2015-08-20 14:36:10 +00:00
|
|
|
fetchCmd.Flags().BoolVarP(&fetchRecentArg, "recent", "r", false, "Fetch recent refs & commits")
|
2015-09-03 11:29:00 +00:00
|
|
|
fetchCmd.Flags().BoolVarP(&fetchAllArg, "all", "a", false, "Fetch all LFS files ever referenced")
|
2015-10-15 15:35:26 +00:00
|
|
|
fetchCmd.Flags().BoolVarP(&fetchPruneArg, "prune", "p", false, "After fetching, prune old data")
|
2015-07-23 16:53:58 +00:00
|
|
|
RootCmd.AddCommand(fetchCmd)
|
|
|
|
}
|
|
|
|
|
2015-08-26 15:13:50 +00:00
|
|
|
func pointersToFetchForRef(ref string) ([]*lfs.WrappedPointer, error) {
|
|
|
|
// Use SkipDeletedBlobs to avoid fetching ALL previous versions of modified files
|
2015-10-28 16:06:36 +00:00
|
|
|
opts := lfs.NewScanRefsOptions()
|
|
|
|
opts.ScanMode = lfs.ScanRefsMode
|
|
|
|
opts.SkipDeletedBlobs = true
|
2015-08-26 15:13:50 +00:00
|
|
|
return lfs.ScanRefs(ref, "", opts)
|
|
|
|
}
|
|
|
|
|
2015-08-07 17:18:39 +00:00
|
|
|
func fetchRefToChan(ref string, include, exclude []string) chan *lfs.WrappedPointer {
|
2015-07-29 09:26:14 +00:00
|
|
|
c := make(chan *lfs.WrappedPointer)
|
2015-08-26 15:13:50 +00:00
|
|
|
pointers, err := pointersToFetchForRef(ref)
|
2015-07-29 09:26:14 +00:00
|
|
|
if err != nil {
|
|
|
|
Panic(err, "Could not scan for Git LFS files")
|
|
|
|
}
|
|
|
|
|
2015-08-07 17:18:39 +00:00
|
|
|
go fetchAndReportToChan(pointers, include, exclude, c)
|
2015-07-29 09:26:14 +00:00
|
|
|
|
|
|
|
return c
|
|
|
|
}
|
|
|
|
|
2015-07-27 10:28:46 +00:00
|
|
|
// Fetch all binaries for a given ref (that we don't have already)
|
2015-10-13 14:29:25 +00:00
|
|
|
func fetchRef(ref string, include, exclude []string) bool {
|
2015-08-26 15:13:50 +00:00
|
|
|
pointers, err := pointersToFetchForRef(ref)
|
2015-07-27 10:28:46 +00:00
|
|
|
if err != nil {
|
|
|
|
Panic(err, "Could not scan for Git LFS files")
|
|
|
|
}
|
2015-10-13 14:29:25 +00:00
|
|
|
return fetchPointers(pointers, include, exclude)
|
2015-07-27 10:28:46 +00:00
|
|
|
}
|
|
|
|
|
2015-08-21 14:19:16 +00:00
|
|
|
// Fetch all previous versions of objects from since to ref (not including final state at ref)
|
|
|
|
// So this will fetch all the '-' sides of the diff from since to ref
|
2015-10-13 14:29:25 +00:00
|
|
|
func fetchPreviousVersions(ref string, since time.Time, include, exclude []string) bool {
|
2015-08-21 14:19:16 +00:00
|
|
|
pointers, err := lfs.ScanPreviousVersions(ref, since)
|
|
|
|
if err != nil {
|
|
|
|
Panic(err, "Could not scan for Git LFS previous versions")
|
|
|
|
}
|
2015-10-13 14:29:25 +00:00
|
|
|
return fetchPointers(pointers, include, exclude)
|
2015-08-21 14:19:16 +00:00
|
|
|
}
|
|
|
|
|
2015-08-20 14:36:10 +00:00
|
|
|
// Fetch recent objects based on config
|
2015-10-13 14:29:25 +00:00
|
|
|
func fetchRecent(alreadyFetchedRefs []*git.Ref, include, exclude []string) bool {
|
2016-05-13 16:38:06 +00:00
|
|
|
fetchconf := config.Config.FetchPruneConfig()
|
2015-08-20 14:36:10 +00:00
|
|
|
|
2015-08-21 14:19:16 +00:00
|
|
|
if fetchconf.FetchRecentRefsDays == 0 && fetchconf.FetchRecentCommitsDays == 0 {
|
2015-10-13 14:29:25 +00:00
|
|
|
return true
|
2015-08-21 14:19:16 +00:00
|
|
|
}
|
|
|
|
|
2015-10-13 14:29:25 +00:00
|
|
|
ok := true
|
2015-08-21 14:19:16 +00:00
|
|
|
// Make a list of what unique commits we've already fetched for to avoid duplicating work
|
|
|
|
uniqueRefShas := make(map[string]string, len(alreadyFetchedRefs))
|
|
|
|
for _, ref := range alreadyFetchedRefs {
|
|
|
|
uniqueRefShas[ref.Sha] = ref.Name
|
|
|
|
}
|
2015-08-20 14:36:10 +00:00
|
|
|
// First find any other recent refs
|
|
|
|
if fetchconf.FetchRecentRefsDays > 0 {
|
2015-08-27 14:18:46 +00:00
|
|
|
Print("Fetching recent branches within %v days", fetchconf.FetchRecentRefsDays)
|
2015-08-20 14:36:10 +00:00
|
|
|
refsSince := time.Now().AddDate(0, 0, -fetchconf.FetchRecentRefsDays)
|
2016-05-13 16:38:06 +00:00
|
|
|
refs, err := git.RecentBranches(refsSince, fetchconf.FetchRecentRefsIncludeRemotes, config.Config.CurrentRemote)
|
2015-08-20 14:36:10 +00:00
|
|
|
if err != nil {
|
|
|
|
Panic(err, "Could not scan for recent refs")
|
|
|
|
}
|
2015-08-21 14:19:16 +00:00
|
|
|
for _, ref := range refs {
|
2015-08-20 14:36:10 +00:00
|
|
|
// Don't fetch for the same SHA twice
|
|
|
|
if prevRefName, ok := uniqueRefShas[ref.Sha]; ok {
|
2015-08-27 14:18:46 +00:00
|
|
|
if ref.Name != prevRefName {
|
|
|
|
tracerx.Printf("Skipping fetch for %v, already fetched via %v", ref.Name, prevRefName)
|
|
|
|
}
|
2015-08-20 14:36:10 +00:00
|
|
|
} else {
|
|
|
|
uniqueRefShas[ref.Sha] = ref.Name
|
2015-08-27 14:18:46 +00:00
|
|
|
Print("Fetching %v", ref.Name)
|
2015-10-13 14:29:25 +00:00
|
|
|
k := fetchRef(ref.Sha, include, exclude)
|
|
|
|
ok = ok && k
|
2015-08-21 14:19:16 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// For every unique commit we've fetched, check recent commits too
|
|
|
|
if fetchconf.FetchRecentCommitsDays > 0 {
|
|
|
|
for commit, refName := range uniqueRefShas {
|
|
|
|
// We measure from the last commit at the ref
|
|
|
|
summ, err := git.GetCommitSummary(commit)
|
|
|
|
if err != nil {
|
|
|
|
Error("Couldn't scan commits at %v: %v", refName, err)
|
|
|
|
continue
|
2015-08-20 14:36:10 +00:00
|
|
|
}
|
2015-08-27 14:18:46 +00:00
|
|
|
Print("Fetching changes within %v days of %v", fetchconf.FetchRecentCommitsDays, refName)
|
2015-08-21 14:19:16 +00:00
|
|
|
commitsSince := summ.CommitDate.AddDate(0, 0, -fetchconf.FetchRecentCommitsDays)
|
2015-10-13 14:29:25 +00:00
|
|
|
k := fetchPreviousVersions(commit, commitsSince, include, exclude)
|
|
|
|
ok = ok && k
|
2015-08-20 14:36:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|
2015-10-13 14:29:25 +00:00
|
|
|
return ok
|
2015-08-20 14:36:10 +00:00
|
|
|
}
|
|
|
|
|
2015-10-13 14:29:25 +00:00
|
|
|
func fetchAll() bool {
|
2015-09-09 22:44:58 +00:00
|
|
|
pointers := scanAll()
|
|
|
|
Print("Fetching objects...")
|
2015-10-13 14:29:25 +00:00
|
|
|
return fetchPointers(pointers, nil, nil)
|
2015-09-09 22:44:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func scanAll() []*lfs.WrappedPointer {
|
2015-09-03 11:29:00 +00:00
|
|
|
// converts to `git rev-list --all`
|
|
|
|
// We only pick up objects in real commits and not the reflog
|
2015-10-28 16:06:36 +00:00
|
|
|
opts := lfs.NewScanRefsOptions()
|
|
|
|
opts.ScanMode = lfs.ScanAllMode
|
|
|
|
opts.SkipDeletedBlobs = false
|
2015-09-09 22:44:58 +00:00
|
|
|
|
2015-09-03 16:23:30 +00:00
|
|
|
// This could be a long process so use the chan version & report progress
|
2015-09-03 16:38:26 +00:00
|
|
|
Print("Scanning for all objects ever referenced...")
|
2016-05-06 13:46:13 +00:00
|
|
|
spinner := progress.NewSpinner()
|
2015-09-03 16:23:30 +00:00
|
|
|
var numObjs int64
|
|
|
|
pointerchan, err := lfs.ScanRefsToChan("", "", opts)
|
2015-09-03 11:29:00 +00:00
|
|
|
if err != nil {
|
|
|
|
Panic(err, "Could not scan for Git LFS files")
|
|
|
|
}
|
2015-09-09 22:44:58 +00:00
|
|
|
|
2015-09-03 16:23:30 +00:00
|
|
|
pointers := make([]*lfs.WrappedPointer, 0)
|
2015-09-09 22:44:58 +00:00
|
|
|
|
2016-03-31 11:00:44 +00:00
|
|
|
for p := range pointerchan.Results {
|
2015-09-03 16:23:30 +00:00
|
|
|
numObjs++
|
|
|
|
spinner.Print(OutputWriter, fmt.Sprintf("%d objects found", numObjs))
|
|
|
|
pointers = append(pointers, p)
|
|
|
|
}
|
2016-03-31 11:00:44 +00:00
|
|
|
err = pointerchan.Wait()
|
|
|
|
if err != nil {
|
|
|
|
Panic(err, "Could not scan for Git LFS files")
|
|
|
|
}
|
2015-09-09 22:44:58 +00:00
|
|
|
|
2015-09-03 16:23:30 +00:00
|
|
|
spinner.Finish(OutputWriter, fmt.Sprintf("%d objects found", numObjs))
|
2015-09-09 22:44:58 +00:00
|
|
|
return pointers
|
2015-09-03 11:29:00 +00:00
|
|
|
}
|
|
|
|
|
2015-10-13 14:29:25 +00:00
|
|
|
func fetchPointers(pointers []*lfs.WrappedPointer, include, exclude []string) bool {
|
|
|
|
return fetchAndReportToChan(pointers, include, exclude, nil)
|
2015-07-23 16:53:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Fetch and report completion of each OID to a channel (optional, pass nil to skip)
|
2015-10-13 14:29:25 +00:00
|
|
|
// Returns true if all completed with no errors, false if errors were written to stderr/log
|
|
|
|
func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []string, out chan<- *lfs.WrappedPointer) bool {
|
2015-07-27 20:41:57 +00:00
|
|
|
totalSize := int64(0)
|
|
|
|
for _, p := range pointers {
|
|
|
|
totalSize += p.Size
|
|
|
|
}
|
2015-07-30 14:43:50 +00:00
|
|
|
q := lfs.NewDownloadQueue(len(pointers), totalSize, false)
|
2015-05-14 16:33:29 +00:00
|
|
|
|
2015-07-23 16:53:58 +00:00
|
|
|
if out != nil {
|
|
|
|
dlwatch := q.Watch()
|
2015-05-27 19:45:18 +00:00
|
|
|
|
|
|
|
go func() {
|
2015-07-23 16:53:58 +00:00
|
|
|
// fetch only reports single OID, but OID *might* be referenced by multiple
|
|
|
|
// WrappedPointers if same content is at multiple paths, so map oid->slice
|
|
|
|
oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers))
|
2015-05-27 19:45:18 +00:00
|
|
|
for _, pointer := range pointers {
|
2015-07-23 16:53:58 +00:00
|
|
|
plist := oidToPointers[pointer.Oid]
|
|
|
|
oidToPointers[pointer.Oid] = append(plist, pointer)
|
2015-05-27 19:45:18 +00:00
|
|
|
}
|
|
|
|
|
2015-07-23 16:53:58 +00:00
|
|
|
for oid := range dlwatch {
|
|
|
|
plist, ok := oidToPointers[oid]
|
2015-05-27 19:45:18 +00:00
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
2015-07-23 16:53:58 +00:00
|
|
|
for _, p := range plist {
|
|
|
|
out <- p
|
2015-05-27 19:45:18 +00:00
|
|
|
}
|
|
|
|
}
|
2015-07-23 16:53:58 +00:00
|
|
|
close(out)
|
2015-05-27 19:45:18 +00:00
|
|
|
}()
|
2016-02-03 19:40:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, p := range pointers {
|
|
|
|
// Only add to download queue if local file is not the right size already
|
|
|
|
// This avoids previous case of over-reporting a requirement for files we already have
|
|
|
|
// which would only be skipped by PointerSmudgeObject later
|
|
|
|
passFilter := lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude)
|
2016-01-31 21:06:40 +00:00
|
|
|
|
2016-04-04 21:09:07 +00:00
|
|
|
lfs.LinkOrCopyFromReference(p.Oid, p.Size)
|
2016-01-31 21:06:40 +00:00
|
|
|
|
2016-02-03 19:40:47 +00:00
|
|
|
if !lfs.ObjectExistsOfSize(p.Oid, p.Size) && passFilter {
|
|
|
|
tracerx.Printf("fetch %v [%v]", p.Name, p.Oid)
|
|
|
|
q.Add(lfs.NewDownloadable(p))
|
|
|
|
} else {
|
|
|
|
if !passFilter {
|
|
|
|
tracerx.Printf("Skipping %v [%v], include/exclude filters applied", p.Name, p.Oid)
|
|
|
|
} else {
|
|
|
|
tracerx.Printf("Skipping %v [%v], already exists", p.Name, p.Oid)
|
|
|
|
}
|
|
|
|
|
|
|
|
// If we already have it, or it won't be fetched
|
|
|
|
// report it to chan immediately to support pull/checkout
|
|
|
|
if out != nil {
|
|
|
|
out <- p
|
|
|
|
}
|
2015-06-20 16:10:54 +00:00
|
|
|
|
2016-02-03 19:40:47 +00:00
|
|
|
}
|
2015-07-23 16:53:58 +00:00
|
|
|
}
|
2016-02-03 19:40:47 +00:00
|
|
|
|
2015-06-20 16:10:54 +00:00
|
|
|
processQueue := time.Now()
|
2015-07-09 18:21:49 +00:00
|
|
|
q.Wait()
|
2015-06-20 16:10:54 +00:00
|
|
|
tracerx.PerformanceSince("process queue", processQueue)
|
2015-08-17 21:45:43 +00:00
|
|
|
|
2015-10-13 14:29:25 +00:00
|
|
|
ok := true
|
2015-08-17 21:45:43 +00:00
|
|
|
for _, err := range q.Errors() {
|
2015-10-13 14:29:25 +00:00
|
|
|
ok = false
|
2016-02-23 16:11:52 +00:00
|
|
|
ExitWithError(err)
|
2015-08-17 21:45:43 +00:00
|
|
|
}
|
2015-10-13 14:29:25 +00:00
|
|
|
return ok
|
2015-05-14 16:33:29 +00:00
|
|
|
}
|