introduce gitscanner callbacks to ScanPreviousVersions()

commands/command_fetch.go

@@ -138,14 +138,23 @@ func fetchRef(gitscanner *lfs.GitScanner, ref string, filter *filepathfilter.Fil
 // Fetch all previous versions of objects from since to ref (not including final state at ref)
 // So this will fetch all the '-' sides of the diff from since to ref
 func fetchPreviousVersions(gitscanner *lfs.GitScanner, ref string, since time.Time, filter *filepathfilter.Filter) bool {
-	pointerCh, err := gitscanner.ScanPreviousVersions(ref, since)
+	var pointers []*lfs.WrappedPointer
+
+	tempgitscanner := lfs.NewGitScanner(nil)
+	err := tempgitscanner.ScanPreviousVersions(ref, since, func(p *lfs.WrappedPointer, err error) {
+		if err != nil {
+			Panic(err, "Could not scan for Git LFS previous versions")
+			return
+		}
+
+		pointers = append(pointers, p)
+	})
+
 	if err != nil {
 		ExitWithError(err)
 	}
-	pointers, err := collectPointers(pointerCh)
-	if err != nil {
-		Panic(err, "Could not scan for Git LFS previous versions")
-	}
+
+	tempgitscanner.Close()
 	return fetchAndReportToChan(pointers, filter, nil)
 }
 

commands/command_prune.go

@@ -323,19 +323,20 @@ func pruneTaskGetRetainedAtRef(gitscanner *lfs.GitScanner, ref string, retainCha
 func pruneTaskGetPreviousVersionsOfRef(gitscanner *lfs.GitScanner, ref string, since time.Time, retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
 	defer waitg.Done()
 
-	refchan, err := gitscanner.ScanPreviousVersions(ref, since)
+	err := gitscanner.ScanPreviousVersions(ref, since, func(p *lfs.WrappedPointer, err error) {
+		if err != nil {
+			errorChan <- err
+			return
+		} else {
+			retainChan <- p.Oid
+			tracerx.Printf("RETAIN: %v via ref %v >= %v", p.Oid, ref, since)
+		}
+	})
+
 	if err != nil {
 		errorChan <- err
 		return
 	}
-	for wp := range refchan.Results {
-		retainChan <- wp.Pointer.Oid
-		tracerx.Printf("RETAIN: %v via ref %v >= %v", wp.Pointer.Oid, ref, since)
-	}
-	err = refchan.Wait()
-	if err != nil {
-		errorChan <- err
-	}
 }
 
 // Background task, must call waitg.Done() once at end

lfs/gitscanner.go

@@ -122,8 +122,12 @@ func (s *GitScanner) ScanUnpushed(remote string, cb GitScannerCallback) error {
 // Returns channel of pointers for *previous* versions that overlap that time.
 // Does not include pointers which were still in use at ref (use ScanRefsToChan
 // for that)
-func (s *GitScanner) ScanPreviousVersions(ref string, since time.Time) (*PointerChannelWrapper, error) {
-	return logPreviousSHAs(ref, since)
+func (s *GitScanner) ScanPreviousVersions(ref string, since time.Time, cb GitScannerCallback) error {
+	callback, err := firstGitScannerCallback(cb, s.callback)
+	if err != nil {
+		return err
+	}
+	return logPreviousSHAs(callback, ref, since)
 }
 
 // ScanIndex scans the git index for modified LFS objects.

lfs/gitscanner_log.go

@@ -58,12 +58,15 @@ func scanUnpushed(cb GitScannerCallback, remote string) error {
 		return err
 	}
 
-	cmd.Stdin.Close()
+	parseScannerLogOutput(cb, LogDiffAdditions, cmd)
+	return nil
+}
 
+func parseScannerLogOutput(cb GitScannerCallback, direction LogDiffDirection, cmd *wrappedCmd) {
 	ch := make(chan gitscannerResult, chanBufSize)
 
 	go func() {
-		scanner := newLogScanner(LogDiffAdditions, cmd.Stdout)
+		scanner := newLogScanner(direction, cmd.Stdout)
 		for scanner.Scan() {
 			if p := scanner.Pointer(); p != nil {
 				ch <- gitscannerResult{Pointer: p}
@@ -77,16 +80,15 @@ func scanUnpushed(cb GitScannerCallback, remote string) error {
 		close(ch)
 	}()
 
+	cmd.Stdin.Close()
 	for result := range ch {
 		cb(result.Pointer, result.Err)
 	}
-
-	return nil
 }
 
 // logPreviousVersions scans history for all previous versions of LFS pointers
 // from 'since' up to (but not including) the final state at ref
-func logPreviousSHAs(ref string, since time.Time) (*PointerChannelWrapper, error) {
+func logPreviousSHAs(cb GitScannerCallback, ref string, since time.Time) error {
 	logArgs := []string{"log",
 		fmt.Sprintf("--since=%v", git.FormatGitDate(since)),
 	}
@@ -97,29 +99,11 @@ func logPreviousSHAs(ref string, since time.Time) (*PointerChannelWrapper, error
 
 	cmd, err := startCommand("git", logArgs...)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	cmd.Stdin.Close()
-
-	pchan := make(chan *WrappedPointer, chanBufSize)
-	errchan := make(chan error, 1)
-
-	// we pull out deletions, since we want the previous SHAs at commits in the range
-	// this means we pick up all previous versions that could have been checked
-	// out in the date range, not just if the commit which *introduced* them is in the range
-	go func() {
-		parseLogOutputToPointers(cmd.Stdout, LogDiffDeletions, nil, nil, pchan)
-		stderr, _ := ioutil.ReadAll(cmd.Stderr)
-		err := cmd.Wait()
-		if err != nil {
-			errchan <- fmt.Errorf("Error in git log: %v %v", err, string(stderr))
-		}
-		close(pchan)
-		close(errchan)
-	}()
-
-	return NewPointerChannelWrapper(pchan, errchan), nil
+	parseScannerLogOutput(cb, LogDiffDeletions, cmd)
+	return nil
 }
 
 func parseLogOutputToPointers(log io.Reader, dir LogDiffDirection,

lfs/scanner_git_test.go

@@ -172,8 +172,8 @@ func TestScanPreviousVersions(t *testing.T) {
 	// where the '-' side of the diff is inside the date range
 
 	// 7 day limit excludes [0] commit, but includes state from that if there
-	// was a subsequent change
-	pointers, err := scanPreviousVersions("master", now.AddDate(0, 0, -7))
+	// was a subsequent chang
+	pointers, err := scanPreviousVersions(t, "master", now.AddDate(0, 0, -7))
 	assert.Equal(t, nil, err)
 
 	// Includes the following 'before' state at commits:
@@ -191,15 +191,16 @@ func TestScanPreviousVersions(t *testing.T) {
 	assert.Equal(t, expected, pointers)
 }
 
-func scanPreviousVersions(ref string, since time.Time) ([]*WrappedPointer, error) {
-	gitscanner := NewGitScanner(nil)
-	pointerchan, err := gitscanner.ScanPreviousVersions(ref, since)
-	if err != nil {
-		return nil, err
-	}
+func scanPreviousVersions(t *testing.T, ref string, since time.Time) ([]*WrappedPointer, error) {
 	pointers := make([]*WrappedPointer, 0, 10)
-	for p := range pointerchan.Results {
+	gitscanner := NewGitScanner(func(p *WrappedPointer, err error) {
+		if err != nil {
+			t.Error(err)
+			return
+		}
 		pointers = append(pointers, p)
-	}
-	return pointers, pointerchan.Wait()
+	})
+
+	err := gitscanner.ScanPreviousVersions(ref, since, nil)
+	return pointers, err
 }