From 9a51ea0f469e3cee6da2d506863bd635f2966e47 Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Fri, 9 Aug 2019 15:25:23 +0000 Subject: [PATCH] Pass environment variables into object scanner We're going to need the environment variables in the object scanner, so pass the appropriate Environment instance down into the object scanner. Use an interface to avoid an import loop between the git and config packages. Note that the environment is not yet used, but will be in a future commit. --- commands/command_checkout.go | 4 ++-- commands/command_fetch.go | 8 ++++---- commands/command_fsck.go | 2 +- commands/command_ls_files.go | 2 +- commands/command_migrate_export.go | 2 +- commands/command_prune.go | 2 +- commands/command_pull.go | 2 +- commands/command_status.go | 4 ++-- commands/uploader.go | 2 +- git/config.go | 7 +++++++ git/object_scanner.go | 2 +- lfs/gitscanner.go | 20 +++++++++++--------- lfs/gitscanner_catfilebatch.go | 9 +++++---- lfs/gitscanner_index.go | 5 +++-- lfs/gitscanner_refs.go | 9 +++++---- lfs/gitscanner_tree.go | 9 +++++---- lfs/scanner.go | 9 ++++++--- lfs/scanner_git_test.go | 5 +++-- 18 files changed, 60 insertions(+), 43 deletions(-) diff --git a/commands/command_checkout.go b/commands/command_checkout.go index fd758603..760c99f0 100644 --- a/commands/command_checkout.go +++ b/commands/command_checkout.go @@ -54,7 +54,7 @@ func checkoutCommand(cmd *cobra.Command, args []string) { meter.Direction = tq.Checkout meter.Logger = meter.LoggerFromEnv(cfg.Os) logger.Enqueue(meter) - chgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { + chgitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) { if err != nil { LoggedError(err, "Scanner error: %s", err) return @@ -99,7 +99,7 @@ func checkoutConflict(file string, stage git.IndexStage) { Exit("Could not checkout (are you not in the middle of a merge?): %v", err) } - scanner, err := git.NewObjectScanner() + scanner, err := git.NewObjectScanner(cfg.OSEnv()) if err != nil { Exit("Could not create object scanner: %v", err) } diff --git a/commands/command_fetch.go b/commands/command_fetch.go index 08ef2631..2cf814f8 100644 --- a/commands/command_fetch.go +++ b/commands/command_fetch.go @@ -60,7 +60,7 @@ func fetchCommand(cmd *cobra.Command, args []string) { } success := true - gitscanner := lfs.NewGitScanner(nil) + gitscanner := lfs.NewGitScanner(cfg, nil) defer gitscanner.Close() include, exclude := getIncludeExcludeArgs(cmd) @@ -110,7 +110,7 @@ func fetchCommand(cmd *cobra.Command, args []string) { func pointersToFetchForRef(ref string, filter *filepathfilter.Filter) ([]*lfs.WrappedPointer, error) { var pointers []*lfs.WrappedPointer var multiErr error - tempgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { + tempgitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) { if err != nil { if multiErr != nil { multiErr = fmt.Errorf("%v\n%v", multiErr, err) @@ -147,7 +147,7 @@ func fetchRef(ref string, filter *filepathfilter.Filter) bool { func fetchPreviousVersions(ref string, since time.Time, filter *filepathfilter.Filter) bool { var pointers []*lfs.WrappedPointer - tempgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { + tempgitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) { if err != nil { Panic(err, "Could not scan for Git LFS previous versions") return @@ -239,7 +239,7 @@ func scanAll() []*lfs.WrappedPointer { // use temp gitscanner to collect pointers var pointers []*lfs.WrappedPointer var multiErr error - tempgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { + tempgitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) { if err != nil { if multiErr != nil { multiErr = fmt.Errorf("%v\n%v", multiErr, err) diff --git a/commands/command_fsck.go b/commands/command_fsck.go index aeb1bb1a..e70f5c32 100644 --- a/commands/command_fsck.go +++ b/commands/command_fsck.go @@ -34,7 +34,7 @@ func fsckCommand(cmd *cobra.Command, args []string) { } var corruptOids []string - gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { + gitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) { if err == nil { var pointerOk bool pointerOk, err = fsckPointer(p.Name, p.Oid) diff --git a/commands/command_ls_files.go b/commands/command_ls_files.go index 1bcbd76c..eee5ad9e 100644 --- a/commands/command_ls_files.go +++ b/commands/command_ls_files.go @@ -62,7 +62,7 @@ func lsFilesCommand(cmd *cobra.Command, args []string) { seen := make(map[string]struct{}) - gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { + gitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) { if err != nil { Exit("Could not scan for Git LFS tree: %s", err) return diff --git a/commands/command_migrate_export.go b/commands/command_migrate_export.go index 33759c29..db329721 100644 --- a/commands/command_migrate_export.go +++ b/commands/command_migrate_export.go @@ -121,7 +121,7 @@ func migrateExportCommand(cmd *cobra.Command, args []string) { // If we have a valid remote, pre-download all objects using the Transfer Queue if remoteURL != "" { q := newDownloadQueue(getTransferManifestOperationRemote("Download", remote), remote) - gs := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { + gs := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) { if err != nil { return } diff --git a/commands/command_prune.go b/commands/command_prune.go index f57dcff1..993a6b1f 100644 --- a/commands/command_prune.go +++ b/commands/command_prune.go @@ -91,7 +91,7 @@ func prune(fetchPruneConfig lfs.FetchPruneConfig, verifyRemote, dryRun, verbose // Now find files to be retained from many sources retainChan := make(chan string, 100) - gitscanner := lfs.NewGitScanner(nil) + gitscanner := lfs.NewGitScanner(cfg, nil) gitscanner.Filter = filepathfilter.New(nil, cfg.FetchExcludePaths()) sem := semaphore.NewWeighted(int64(runtime.NumCPU() * 2)) diff --git a/commands/command_pull.go b/commands/command_pull.go index 6d19e03f..a20dbd52 100644 --- a/commands/command_pull.go +++ b/commands/command_pull.go @@ -47,7 +47,7 @@ func pull(filter *filepathfilter.Filter) { remote := cfg.Remote() singleCheckout := newSingleCheckout(cfg.Git, remote) q := newDownloadQueue(singleCheckout.Manifest(), remote, tq.WithProgress(meter)) - gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { + gitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) { if err != nil { LoggedError(err, "Scanner error: %s", err) return diff --git a/commands/command_status.go b/commands/command_status.go index 184b1890..f3b0e594 100644 --- a/commands/command_status.go +++ b/commands/command_status.go @@ -32,7 +32,7 @@ func statusCommand(cmd *cobra.Command, args []string) { scanIndexAt = git.RefBeforeFirstCommit } - scanner, err := lfs.NewPointerScanner() + scanner, err := lfs.NewPointerScanner(cfg.OSEnv()) if err != nil { ExitWithError(err) } @@ -228,7 +228,7 @@ func statusScanRefRange(ref *git.Ref) { return } - gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { + gitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) { if err != nil { Panic(err, "Could not scan for Git LFS objects") return diff --git a/commands/uploader.go b/commands/uploader.go index fa10f738..5e77adeb 100644 --- a/commands/uploader.go +++ b/commands/uploader.go @@ -151,7 +151,7 @@ func (c *uploadContext) addScannerError(err error) { } func (c *uploadContext) buildGitScanner() (*lfs.GitScanner, error) { - gitscanner := lfs.NewGitScanner(nil) + gitscanner := lfs.NewGitScanner(cfg, nil) gitscanner.FoundLockable = func(n string) { c.lockVerifier.LockedByThem(n) } gitscanner.PotentialLockables = c.lockVerifier return gitscanner, gitscanner.RemoteForPush(c.Remote) diff --git a/git/config.go b/git/config.go index 9f22faa5..c9840dab 100644 --- a/git/config.go +++ b/git/config.go @@ -14,6 +14,13 @@ var ( ErrReadOnly = errors.New("configuration is read-only") ) +// Environment is a restricted version of config.Environment that only provides +// a single method. +type Environment interface { + // Get is shorthand for calling `e.Fetcher.Get(key)`. + Get(key string) (val string, ok bool) +} + // Configuration can fetch or modify the current Git config and track the Git // version. type Configuration struct { diff --git a/git/object_scanner.go b/git/object_scanner.go index 5d92be88..7827007f 100644 --- a/git/object_scanner.go +++ b/git/object_scanner.go @@ -42,7 +42,7 @@ type ObjectScanner struct { // command, they will be returned immediately. // // Otherwise, an `*ObjectScanner` is returned with no error. -func NewObjectScanner() (*ObjectScanner, error) { +func NewObjectScanner(osEnv Environment) (*ObjectScanner, error) { gitdir, err := GitCommonDir() if err != nil { return nil, err diff --git a/lfs/gitscanner.go b/lfs/gitscanner.go index 69238db2..e06dd90f 100644 --- a/lfs/gitscanner.go +++ b/lfs/gitscanner.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/git-lfs/git-lfs/config" "github.com/git-lfs/git-lfs/filepathfilter" "github.com/rubyist/tracerx" ) @@ -30,6 +31,7 @@ type GitScanner struct { closed bool started time.Time mu sync.Mutex + cfg *config.Configuration } type GitScannerFoundPointer func(*WrappedPointer, error) @@ -41,8 +43,8 @@ type GitScannerSet interface { // NewGitScanner initializes a *GitScanner for a Git repository in the current // working directory. -func NewGitScanner(cb GitScannerFoundPointer) *GitScanner { - return &GitScanner{started: time.Now(), FoundPointer: cb} +func NewGitScanner(cfg *config.Configuration, cb GitScannerFoundPointer) *GitScanner { + return &GitScanner{started: time.Now(), FoundPointer: cb, cfg: cfg} } // Close stops exits once all processing has stopped, and all resources are @@ -90,7 +92,7 @@ func (s *GitScanner) ScanRangeToRemote(left, right string, cb GitScannerFoundPoi } s.mu.Unlock() - return scanLeftRightToChan(s, callback, left, right, s.opts(ScanRangeToRemoteMode)) + return scanLeftRightToChan(s, callback, left, right, s.cfg.OSEnv(), s.opts(ScanRangeToRemoteMode)) } // ScanRefs through all commits reachable by refs contained in "include" and @@ -103,7 +105,7 @@ func (s *GitScanner) ScanRefs(include, exclude []string, cb GitScannerFoundPoint opts := s.opts(ScanRefsMode) opts.SkipDeletedBlobs = false - return scanRefsToChan(s, callback, include, exclude, opts) + return scanRefsToChan(s, callback, include, exclude, s.cfg.OSEnv(), opts) } // ScanRefRange scans through all commits from the given left and right refs, @@ -116,7 +118,7 @@ func (s *GitScanner) ScanRefRange(left, right string, cb GitScannerFoundPointer) opts := s.opts(ScanRefsMode) opts.SkipDeletedBlobs = false - return scanLeftRightToChan(s, callback, left, right, opts) + return scanLeftRightToChan(s, callback, left, right, s.cfg.OSEnv(), opts) } // ScanRefWithDeleted scans through all objects in the given ref, including @@ -135,7 +137,7 @@ func (s *GitScanner) ScanRef(ref string, cb GitScannerFoundPointer) error { opts := s.opts(ScanRefsMode) opts.SkipDeletedBlobs = true - return scanLeftRightToChan(s, callback, ref, "", opts) + return scanLeftRightToChan(s, callback, ref, "", s.cfg.OSEnv(), opts) } // ScanAll scans through all objects in the git repository. @@ -147,7 +149,7 @@ func (s *GitScanner) ScanAll(cb GitScannerFoundPointer) error { opts := s.opts(ScanAllMode) opts.SkipDeletedBlobs = false - return scanLeftRightToChan(s, callback, "", "", opts) + return scanLeftRightToChan(s, callback, "", "", s.cfg.OSEnv(), opts) } // ScanTree takes a ref and returns WrappedPointer objects in the tree at that @@ -158,7 +160,7 @@ func (s *GitScanner) ScanTree(ref string) error { if err != nil { return err } - return runScanTree(callback, ref, s.Filter) + return runScanTree(callback, ref, s.Filter, s.cfg.OSEnv()) } // ScanUnpushed scans history for all LFS pointers which have been added but not @@ -189,7 +191,7 @@ func (s *GitScanner) ScanIndex(ref string, cb GitScannerFoundPointer) error { if err != nil { return err } - return scanIndex(callback, ref, s.Filter) + return scanIndex(callback, ref, s.Filter, s.cfg.OSEnv()) } func (s *GitScanner) opts(mode ScanningMode) *ScanRefsOptions { diff --git a/lfs/gitscanner_catfilebatch.go b/lfs/gitscanner_catfilebatch.go index 9a55ac94..9a66471f 100644 --- a/lfs/gitscanner_catfilebatch.go +++ b/lfs/gitscanner_catfilebatch.go @@ -6,6 +6,7 @@ import ( "fmt" "io" + "github.com/git-lfs/git-lfs/config" "github.com/git-lfs/git-lfs/git" ) @@ -16,8 +17,8 @@ import ( // pointerCh. If a Git Blob is not an LFS pointer, check the lockableSet to see // if that blob is for a locked file. Any errors are sent to errCh. An error is // returned if the 'git cat-file' command fails to start. -func runCatFileBatch(pointerCh chan *WrappedPointer, lockableCh chan string, lockableSet *lockableNameSet, revs *StringChannelWrapper, errCh chan error) error { - scanner, err := NewPointerScanner() +func runCatFileBatch(pointerCh chan *WrappedPointer, lockableCh chan string, lockableSet *lockableNameSet, revs *StringChannelWrapper, errCh chan error, osEnv config.Environment) error { + scanner, err := NewPointerScanner(osEnv) if err != nil { return err } @@ -69,8 +70,8 @@ type PointerScanner struct { err error } -func NewPointerScanner() (*PointerScanner, error) { - scanner, err := git.NewObjectScanner() +func NewPointerScanner(osEnv config.Environment) (*PointerScanner, error) { + scanner, err := git.NewObjectScanner(osEnv) if err != nil { return nil, err } diff --git a/lfs/gitscanner_index.go b/lfs/gitscanner_index.go index 8d3afb0d..801bc827 100644 --- a/lfs/gitscanner_index.go +++ b/lfs/gitscanner_index.go @@ -4,6 +4,7 @@ import ( "strings" "sync" + "github.com/git-lfs/git-lfs/config" "github.com/git-lfs/git-lfs/filepathfilter" ) @@ -12,7 +13,7 @@ import ( // // Ref is the ref at which to scan, which may be "HEAD" if there is at least one // commit. -func scanIndex(cb GitScannerFoundPointer, ref string, f *filepathfilter.Filter) error { +func scanIndex(cb GitScannerFoundPointer, ref string, f *filepathfilter.Filter, osEnv config.Environment) error { indexMap := &indexFileMap{ nameMap: make(map[string][]*indexFile), nameShaPairs: make(map[string]bool), @@ -67,7 +68,7 @@ func scanIndex(cb GitScannerFoundPointer, ref string, f *filepathfilter.Filter) ch := make(chan gitscannerResult, chanBufSize) - barePointerCh, _, err := catFileBatch(smallShas, nil) + barePointerCh, _, err := catFileBatch(smallShas, nil, osEnv) if err != nil { return err } diff --git a/lfs/gitscanner_refs.go b/lfs/gitscanner_refs.go index 89f8f639..d8117fa0 100644 --- a/lfs/gitscanner_refs.go +++ b/lfs/gitscanner_refs.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "regexp" + "github.com/git-lfs/git-lfs/config" "github.com/git-lfs/git-lfs/git" ) @@ -37,7 +38,7 @@ func noopFoundLockable(name string) {} // "include" and not reachable by any refs included in "excluded" and returns // a channel of WrappedPointer objects for all Git LFS pointers it finds. // Reports unique oids once only, not multiple times if >1 file uses the same content -func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, include, exclude []string, opt *ScanRefsOptions) error { +func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, include, exclude []string, osEnv config.Environment, opt *ScanRefsOptions) error { if opt == nil { panic("no scan ref options") } @@ -64,7 +65,7 @@ func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, inclu } }(lockableCb, batchLockableCh) - pointers, checkLockableCh, err := catFileBatch(smallShas, lockableSet) + pointers, checkLockableCh, err := catFileBatch(smallShas, lockableSet, osEnv) if err != nil { return err } @@ -95,8 +96,8 @@ func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, inclu // scanLeftRightToChan takes a ref and returns a channel of WrappedPointer objects // for all Git LFS pointers it finds for that ref. // Reports unique oids once only, not multiple times if >1 file uses the same content -func scanLeftRightToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, refLeft, refRight string, opt *ScanRefsOptions) error { - return scanRefsToChan(scanner, pointerCb, []string{refLeft}, []string{refRight}, opt) +func scanLeftRightToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, refLeft, refRight string, osEnv config.Environment, opt *ScanRefsOptions) error { + return scanRefsToChan(scanner, pointerCb, []string{refLeft}, []string{refRight}, osEnv, opt) } // revListShas uses git rev-list to return the list of object sha1s diff --git a/lfs/gitscanner_tree.go b/lfs/gitscanner_tree.go index a841328a..61f62507 100644 --- a/lfs/gitscanner_tree.go +++ b/lfs/gitscanner_tree.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "github.com/git-lfs/git-lfs/config" "github.com/git-lfs/git-lfs/filepathfilter" "github.com/git-lfs/git-lfs/git" ) @@ -19,7 +20,7 @@ type TreeBlob struct { Filename string } -func runScanTree(cb GitScannerFoundPointer, ref string, filter *filepathfilter.Filter) error { +func runScanTree(cb GitScannerFoundPointer, ref string, filter *filepathfilter.Filter, osEnv config.Environment) error { // We don't use the nameMap approach here since that's imprecise when >1 file // can be using the same content treeShas, err := lsTreeBlobs(ref, filter) @@ -27,7 +28,7 @@ func runScanTree(cb GitScannerFoundPointer, ref string, filter *filepathfilter.F return err } - pcw, err := catFileBatchTree(treeShas) + pcw, err := catFileBatchTree(treeShas, osEnv) if err != nil { return err } @@ -46,8 +47,8 @@ func runScanTree(cb GitScannerFoundPointer, ref string, filter *filepathfilter.F // of a git object, given its sha1. The contents will be decoded into // a Git LFS pointer. treeblobs is a channel over which blob entries // will be sent. It returns a channel from which point.Pointers can be read. -func catFileBatchTree(treeblobs *TreeBlobChannelWrapper) (*PointerChannelWrapper, error) { - scanner, err := NewPointerScanner() +func catFileBatchTree(treeblobs *TreeBlobChannelWrapper, osEnv config.Environment) (*PointerChannelWrapper, error) { + scanner, err := NewPointerScanner(osEnv) if err != nil { return nil, err } diff --git a/lfs/scanner.go b/lfs/scanner.go index 8c79d099..e7c99779 100644 --- a/lfs/scanner.go +++ b/lfs/scanner.go @@ -1,6 +1,9 @@ package lfs -import "github.com/git-lfs/git-lfs/tools" +import ( + "github.com/git-lfs/git-lfs/config" + "github.com/git-lfs/git-lfs/tools" +) const ( // blobSizeCutoff is used to determine which files to scan for Git LFS @@ -45,11 +48,11 @@ func catFileBatchCheck(revs *StringChannelWrapper, lockableSet *lockableNameSet) // of a git object, given its sha1. The contents will be decoded into // a Git LFS pointer. revs is a channel over which strings containing Git SHA1s // will be sent. It returns a channel from which point.Pointers can be read. -func catFileBatch(revs *StringChannelWrapper, lockableSet *lockableNameSet) (*PointerChannelWrapper, chan string, error) { +func catFileBatch(revs *StringChannelWrapper, lockableSet *lockableNameSet, osEnv config.Environment) (*PointerChannelWrapper, chan string, error) { pointerCh := make(chan *WrappedPointer, chanBufSize) lockableCh := make(chan string, chanBufSize) errCh := make(chan error, 5) // shared by 2 goroutines & may add more detail errors? - if err := runCatFileBatch(pointerCh, lockableCh, lockableSet, revs, errCh); err != nil { + if err := runCatFileBatch(pointerCh, lockableCh, lockableSet, revs, errCh, osEnv); err != nil { return nil, nil, err } return NewPointerChannelWrapper(pointerCh, errCh), lockableCh, nil diff --git a/lfs/scanner_git_test.go b/lfs/scanner_git_test.go index e70c95a4..6374ea13 100644 --- a/lfs/scanner_git_test.go +++ b/lfs/scanner_git_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/git-lfs/git-lfs/config" . "github.com/git-lfs/git-lfs/lfs" test "github.com/git-lfs/git-lfs/t/cmd/util" "github.com/stretchr/testify/assert" @@ -91,7 +92,7 @@ func scanUnpushed(remoteName string) ([]*WrappedPointer, error) { pointers := make([]*WrappedPointer, 0, 10) var multiErr error - gitscanner := NewGitScanner(func(p *WrappedPointer, err error) { + gitscanner := NewGitScanner(config.New(), func(p *WrappedPointer, err error) { if err != nil { if multiErr != nil { multiErr = fmt.Errorf("%v\n%v", multiErr, err) @@ -193,7 +194,7 @@ func TestScanPreviousVersions(t *testing.T) { func scanPreviousVersions(t *testing.T, ref string, since time.Time) ([]*WrappedPointer, error) { pointers := make([]*WrappedPointer, 0, 10) - gitscanner := NewGitScanner(func(p *WrappedPointer, err error) { + gitscanner := NewGitScanner(config.New(), func(p *WrappedPointer, err error) { if err != nil { t.Error(err) return