package commands import ( "time" "github.com/github/git-lfs/git" "github.com/github/git-lfs/lfs" "github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx" "github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra" ) var ( fetchCmd = &cobra.Command{ Use: "fetch", Short: "Downloads LFS files", Run: fetchCommand, } fetchIncludeArg string fetchExcludeArg string fetchRecentArg bool ) func fetchCommand(cmd *cobra.Command, args []string) { var refs []*Ref if len(args) > 0 { // Remote is first arg lfs.Config.CurrentRemote = args[0] } else { trackedRemote, err := git.CurrentRemote() if err == nil { lfs.Config.CurrentRemote = trackedRemote } // otherwise leave as default (origin) } if len(args) > 1 { refs = args[1:] } else { ref, err := git.CurrentRef() if err != nil { Panic(err, "Could not fetch") } refs = []*Ref{ref} } includePaths, excludePaths := determineIncludeExcludePaths(fetchIncludeArg, fetchExcludeArg) // Fetch refs sequentially per arg order; duplicates in later refs will be ignored for _, ref := range refs { Print("Fetching %v", ref.Name) fetchRef(ref.Sha, includePaths, excludePaths) } if fetchRecentArg || lfs.Config.FetchPruneConfig().FetchRecentAlways { Print("Fetching recent objects") fetchRecent(refs, includePaths, excludePaths) } } func init() { fetchCmd.Flags().StringVarP(&fetchIncludeArg, "include", "I", "", "Include a list of paths") fetchCmd.Flags().StringVarP(&fetchExcludeArg, "exclude", "X", "", "Exclude a list of paths") fetchCmd.Flags().BoolVarP(&fetchRecentArg, "recent", "r", false, "Fetch recent refs & commits") RootCmd.AddCommand(fetchCmd) } func fetchRefToChan(ref string, include, exclude []string) chan *lfs.WrappedPointer { c := make(chan *lfs.WrappedPointer) pointers, err := lfs.ScanRefs(ref, "", nil) if err != nil { Panic(err, "Could not scan for Git LFS files") } go fetchAndReportToChan(pointers, include, exclude, c) return c } // Fetch all binaries for a given ref (that we don't have already) func fetchRef(ref string, include, exclude []string) { pointers, err := lfs.ScanRefs(ref, "", nil) if err != nil { Panic(err, "Could not scan for Git LFS files") } fetchPointers(pointers, include, exclude) } // Fetch recent objects based on config func fetchRecent(alreadyFetchedRefs []*Ref, include, exclude []string) { fetchconf := lfs.Config.FetchPruneConfig() // First find any other recent refs if fetchconf.FetchRecentRefsDays > 0 { refsSince := time.Now().AddDate(0, 0, -fetchconf.FetchRecentRefsDays) refs, err := git.RecentRefs(refsSince, fetchconf.FetchRecentRefsIncludeRemotes, "") if err != nil { Panic(err, "Could not scan for recent refs") } uniqueRefShas := make(map[string]string, len(refsSince)) for _, ref := range alreadyFetchedRefs { uniqueRefShas[ref.Sha] = ref.Name } for _, ref := range refsSince { // Don't fetch for the same SHA twice if prevRefName, ok := uniqueRefShas[ref.Sha]; ok { Print("Skipping fetch for %v, already fetched commit via %v", ref.Name, prevRefName) } else { uniqueRefShas[ref.Sha] = ref.Name Print("Fetching recent ref %v", ref.Name) fetchRef(ref.Sha, includePaths, excludePaths) } } // Add to list to scan for recent commits } } func fetchPointers(pointers []*lfs.WrappedPointer, include, exclude []string) { fetchAndReportToChan(pointers, include, exclude, nil) } // Fetch and report completion of each OID to a channel (optional, pass nil to skip) func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []string, out chan<- *lfs.WrappedPointer) { totalSize := int64(0) for _, p := range pointers { totalSize += p.Size } q := lfs.NewDownloadQueue(len(pointers), totalSize, false) for _, p := range pointers { // Only add to download queue if local file is not the right size already // This avoids previous case of over-reporting a requirement for files we already have // which would only be skipped by PointerSmudgeObject later passFilter := lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude) if !lfs.ObjectExistsOfSize(p.Oid, p.Size) && passFilter { q.Add(lfs.NewDownloadable(p)) } else { // If we already have it, or it won't be fetched // report it to chan immediately to support pull/checkout if out != nil { out <- p } } } if out != nil { dlwatch := q.Watch() go func() { // fetch only reports single OID, but OID *might* be referenced by multiple // WrappedPointers if same content is at multiple paths, so map oid->slice oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers)) for _, pointer := range pointers { plist := oidToPointers[pointer.Oid] oidToPointers[pointer.Oid] = append(plist, pointer) } for oid := range dlwatch { plist, ok := oidToPointers[oid] if !ok { continue } for _, p := range plist { out <- p } } close(out) }() } processQueue := time.Now() q.Wait() tracerx.PerformanceSince("process queue", processQueue) }