From d8ca6109396e37a927fe6c4ad296120f8bde6c66 Mon Sep 17 00:00:00 2001 From: risk danger olson Date: Tue, 29 Nov 2016 10:27:42 -0700 Subject: [PATCH] introduce gitscanner callbacks to ScanPreviousVersions() --- commands/command_fetch.go | 19 ++++++++++++++----- commands/command_prune.go | 19 ++++++++++--------- lfs/gitscanner.go | 8 ++++++-- lfs/gitscanner_log.go | 36 ++++++++++-------------------------- lfs/scanner_git_test.go | 23 ++++++++++++----------- 5 files changed, 52 insertions(+), 53 deletions(-) diff --git a/commands/command_fetch.go b/commands/command_fetch.go index b3d9b366..3154a947 100644 --- a/commands/command_fetch.go +++ b/commands/command_fetch.go @@ -138,14 +138,23 @@ func fetchRef(gitscanner *lfs.GitScanner, ref string, filter *filepathfilter.Fil // Fetch all previous versions of objects from since to ref (not including final state at ref) // So this will fetch all the '-' sides of the diff from since to ref func fetchPreviousVersions(gitscanner *lfs.GitScanner, ref string, since time.Time, filter *filepathfilter.Filter) bool { - pointerCh, err := gitscanner.ScanPreviousVersions(ref, since) + var pointers []*lfs.WrappedPointer + + tempgitscanner := lfs.NewGitScanner(nil) + err := tempgitscanner.ScanPreviousVersions(ref, since, func(p *lfs.WrappedPointer, err error) { + if err != nil { + Panic(err, "Could not scan for Git LFS previous versions") + return + } + + pointers = append(pointers, p) + }) + if err != nil { ExitWithError(err) } - pointers, err := collectPointers(pointerCh) - if err != nil { - Panic(err, "Could not scan for Git LFS previous versions") - } + + tempgitscanner.Close() return fetchAndReportToChan(pointers, filter, nil) } diff --git a/commands/command_prune.go b/commands/command_prune.go index b6592595..1b9a01b8 100644 --- a/commands/command_prune.go +++ b/commands/command_prune.go @@ -323,19 +323,20 @@ func pruneTaskGetRetainedAtRef(gitscanner *lfs.GitScanner, ref string, retainCha func pruneTaskGetPreviousVersionsOfRef(gitscanner *lfs.GitScanner, ref string, since time.Time, retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) { defer waitg.Done() - refchan, err := gitscanner.ScanPreviousVersions(ref, since) + err := gitscanner.ScanPreviousVersions(ref, since, func(p *lfs.WrappedPointer, err error) { + if err != nil { + errorChan <- err + return + } else { + retainChan <- p.Oid + tracerx.Printf("RETAIN: %v via ref %v >= %v", p.Oid, ref, since) + } + }) + if err != nil { errorChan <- err return } - for wp := range refchan.Results { - retainChan <- wp.Pointer.Oid - tracerx.Printf("RETAIN: %v via ref %v >= %v", wp.Pointer.Oid, ref, since) - } - err = refchan.Wait() - if err != nil { - errorChan <- err - } } // Background task, must call waitg.Done() once at end diff --git a/lfs/gitscanner.go b/lfs/gitscanner.go index 361446a8..24d65a8a 100644 --- a/lfs/gitscanner.go +++ b/lfs/gitscanner.go @@ -122,8 +122,12 @@ func (s *GitScanner) ScanUnpushed(remote string, cb GitScannerCallback) error { // Returns channel of pointers for *previous* versions that overlap that time. // Does not include pointers which were still in use at ref (use ScanRefsToChan // for that) -func (s *GitScanner) ScanPreviousVersions(ref string, since time.Time) (*PointerChannelWrapper, error) { - return logPreviousSHAs(ref, since) +func (s *GitScanner) ScanPreviousVersions(ref string, since time.Time, cb GitScannerCallback) error { + callback, err := firstGitScannerCallback(cb, s.callback) + if err != nil { + return err + } + return logPreviousSHAs(callback, ref, since) } // ScanIndex scans the git index for modified LFS objects. diff --git a/lfs/gitscanner_log.go b/lfs/gitscanner_log.go index 314c625b..ad6708b6 100644 --- a/lfs/gitscanner_log.go +++ b/lfs/gitscanner_log.go @@ -58,12 +58,15 @@ func scanUnpushed(cb GitScannerCallback, remote string) error { return err } - cmd.Stdin.Close() + parseScannerLogOutput(cb, LogDiffAdditions, cmd) + return nil +} +func parseScannerLogOutput(cb GitScannerCallback, direction LogDiffDirection, cmd *wrappedCmd) { ch := make(chan gitscannerResult, chanBufSize) go func() { - scanner := newLogScanner(LogDiffAdditions, cmd.Stdout) + scanner := newLogScanner(direction, cmd.Stdout) for scanner.Scan() { if p := scanner.Pointer(); p != nil { ch <- gitscannerResult{Pointer: p} @@ -77,16 +80,15 @@ func scanUnpushed(cb GitScannerCallback, remote string) error { close(ch) }() + cmd.Stdin.Close() for result := range ch { cb(result.Pointer, result.Err) } - - return nil } // logPreviousVersions scans history for all previous versions of LFS pointers // from 'since' up to (but not including) the final state at ref -func logPreviousSHAs(ref string, since time.Time) (*PointerChannelWrapper, error) { +func logPreviousSHAs(cb GitScannerCallback, ref string, since time.Time) error { logArgs := []string{"log", fmt.Sprintf("--since=%v", git.FormatGitDate(since)), } @@ -97,29 +99,11 @@ func logPreviousSHAs(ref string, since time.Time) (*PointerChannelWrapper, error cmd, err := startCommand("git", logArgs...) if err != nil { - return nil, err + return err } - cmd.Stdin.Close() - - pchan := make(chan *WrappedPointer, chanBufSize) - errchan := make(chan error, 1) - - // we pull out deletions, since we want the previous SHAs at commits in the range - // this means we pick up all previous versions that could have been checked - // out in the date range, not just if the commit which *introduced* them is in the range - go func() { - parseLogOutputToPointers(cmd.Stdout, LogDiffDeletions, nil, nil, pchan) - stderr, _ := ioutil.ReadAll(cmd.Stderr) - err := cmd.Wait() - if err != nil { - errchan <- fmt.Errorf("Error in git log: %v %v", err, string(stderr)) - } - close(pchan) - close(errchan) - }() - - return NewPointerChannelWrapper(pchan, errchan), nil + parseScannerLogOutput(cb, LogDiffDeletions, cmd) + return nil } func parseLogOutputToPointers(log io.Reader, dir LogDiffDirection, diff --git a/lfs/scanner_git_test.go b/lfs/scanner_git_test.go index 9a2f2949..a5cc1eb3 100644 --- a/lfs/scanner_git_test.go +++ b/lfs/scanner_git_test.go @@ -172,8 +172,8 @@ func TestScanPreviousVersions(t *testing.T) { // where the '-' side of the diff is inside the date range // 7 day limit excludes [0] commit, but includes state from that if there - // was a subsequent change - pointers, err := scanPreviousVersions("master", now.AddDate(0, 0, -7)) + // was a subsequent chang + pointers, err := scanPreviousVersions(t, "master", now.AddDate(0, 0, -7)) assert.Equal(t, nil, err) // Includes the following 'before' state at commits: @@ -191,15 +191,16 @@ func TestScanPreviousVersions(t *testing.T) { assert.Equal(t, expected, pointers) } -func scanPreviousVersions(ref string, since time.Time) ([]*WrappedPointer, error) { - gitscanner := NewGitScanner(nil) - pointerchan, err := gitscanner.ScanPreviousVersions(ref, since) - if err != nil { - return nil, err - } +func scanPreviousVersions(t *testing.T, ref string, since time.Time) ([]*WrappedPointer, error) { pointers := make([]*WrappedPointer, 0, 10) - for p := range pointerchan.Results { + gitscanner := NewGitScanner(func(p *WrappedPointer, err error) { + if err != nil { + t.Error(err) + return + } pointers = append(pointers, p) - } - return pointers, pointerchan.Wait() + }) + + err := gitscanner.ScanPreviousVersions(ref, since, nil) + return pointers, err }