lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pass environment variables into object scanner

Browse Source 
We're going to need the environment variables in the object scanner, so
pass the appropriate Environment instance down into the object scanner.
Use an interface to avoid an import loop between the git and config
packages.

Note that the environment is not yet used, but will be in a future
commit.
This commit is contained in:
brian m. carlson 2019-08-09 15:25:23 +00:00
parent bce5ad9af0
commit 9a51ea0f46
No known key found for this signature in database
GPG Key ID: 2D0C9BC12F82B3A1
18 changed files with 60 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
commands/command_checkout.go
View File

@ -54,7 +54,7 @@ func checkoutCommand(cmd *cobra.Command, args []string) {
meter.Direction = tq.Checkout
meter.Logger = meter.LoggerFromEnv(cfg.Os)
logger.Enqueue(meter)
chgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
chgitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) {
if err != nil {
LoggedError(err, "Scanner error: %s", err)
return
@ -99,7 +99,7 @@ func checkoutConflict(file string, stage git.IndexStage) {
Exit("Could not checkout (are you not in the middle of a merge?): %v", err)
}
scanner, err := git.NewObjectScanner()
scanner, err := git.NewObjectScanner(cfg.OSEnv())
if err != nil {
Exit("Could not create object scanner: %v", err)
}

8
commands/command_fetch.go
View File

@ -60,7 +60,7 @@ func fetchCommand(cmd *cobra.Command, args []string) {
}
success := true
gitscanner := lfs.NewGitScanner(nil)
gitscanner := lfs.NewGitScanner(cfg, nil)
defer gitscanner.Close()
include, exclude := getIncludeExcludeArgs(cmd)
@ -110,7 +110,7 @@ func fetchCommand(cmd *cobra.Command, args []string) {
func pointersToFetchForRef(ref string, filter *filepathfilter.Filter) ([]*lfs.WrappedPointer, error) {
var pointers []*lfs.WrappedPointer
var multiErr error
tempgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
tempgitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) {
if err != nil {
if multiErr != nil {
multiErr = fmt.Errorf("%v\n%v", multiErr, err)
@ -147,7 +147,7 @@ func fetchRef(ref string, filter *filepathfilter.Filter) bool {
func fetchPreviousVersions(ref string, since time.Time, filter *filepathfilter.Filter) bool {
var pointers []*lfs.WrappedPointer
tempgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
tempgitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) {
if err != nil {
Panic(err, "Could not scan for Git LFS previous versions")
return
@ -239,7 +239,7 @@ func scanAll() []*lfs.WrappedPointer {
// use temp gitscanner to collect pointers
var pointers []*lfs.WrappedPointer
var multiErr error
tempgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
tempgitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) {
if err != nil {
if multiErr != nil {
multiErr = fmt.Errorf("%v\n%v", multiErr, err)

2
commands/command_fsck.go
View File

@ -34,7 +34,7 @@ func fsckCommand(cmd *cobra.Command, args []string) {
}
var corruptOids []string
gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
gitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) {
if err == nil {
var pointerOk bool
pointerOk, err = fsckPointer(p.Name, p.Oid)

2
commands/command_ls_files.go
View File

@ -62,7 +62,7 @@ func lsFilesCommand(cmd *cobra.Command, args []string) {
seen := make(map[string]struct{})
gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
gitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) {
if err != nil {
Exit("Could not scan for Git LFS tree: %s", err)
return

2
commands/command_migrate_export.go
View File

@ -121,7 +121,7 @@ func migrateExportCommand(cmd *cobra.Command, args []string) {
// If we have a valid remote, pre-download all objects using the Transfer Queue
if remoteURL != "" {
q := newDownloadQueue(getTransferManifestOperationRemote("Download", remote), remote)
gs := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
gs := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) {
if err != nil {
return
}

2
commands/command_prune.go
View File

@ -91,7 +91,7 @@ func prune(fetchPruneConfig lfs.FetchPruneConfig, verifyRemote, dryRun, verbose
// Now find files to be retained from many sources
retainChan := make(chan string, 100)
gitscanner := lfs.NewGitScanner(nil)
gitscanner := lfs.NewGitScanner(cfg, nil)
gitscanner.Filter = filepathfilter.New(nil, cfg.FetchExcludePaths())
sem := semaphore.NewWeighted(int64(runtime.NumCPU() * 2))

2
commands/command_pull.go
View File

@ -47,7 +47,7 @@ func pull(filter *filepathfilter.Filter) {
remote := cfg.Remote()
singleCheckout := newSingleCheckout(cfg.Git, remote)
q := newDownloadQueue(singleCheckout.Manifest(), remote, tq.WithProgress(meter))
gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
gitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) {
if err != nil {
LoggedError(err, "Scanner error: %s", err)
return

4
commands/command_status.go
View File

@ -32,7 +32,7 @@ func statusCommand(cmd *cobra.Command, args []string) {
scanIndexAt = git.RefBeforeFirstCommit
}
scanner, err := lfs.NewPointerScanner()
scanner, err := lfs.NewPointerScanner(cfg.OSEnv())
if err != nil {
ExitWithError(err)
}
@ -228,7 +228,7 @@ func statusScanRefRange(ref *git.Ref) {
return
}
gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
gitscanner := lfs.NewGitScanner(cfg, func(p *lfs.WrappedPointer, err error) {
if err != nil {
Panic(err, "Could not scan for Git LFS objects")
return

2
commands/uploader.go
View File

@ -151,7 +151,7 @@ func (c *uploadContext) addScannerError(err error) {
}
func (c *uploadContext) buildGitScanner() (*lfs.GitScanner, error) {
gitscanner := lfs.NewGitScanner(nil)
gitscanner := lfs.NewGitScanner(cfg, nil)
gitscanner.FoundLockable = func(n string) { c.lockVerifier.LockedByThem(n) }
gitscanner.PotentialLockables = c.lockVerifier
return gitscanner, gitscanner.RemoteForPush(c.Remote)

7
git/config.go
View File

@ -14,6 +14,13 @@ var (
ErrReadOnly = errors.New("configuration is read-only")
)
// Environment is a restricted version of config.Environment that only provides
// a single method.
type Environment interface {
// Get is shorthand for calling `e.Fetcher.Get(key)`.
Get(key string) (val string, ok bool)
}
// Configuration can fetch or modify the current Git config and track the Git
// version.
type Configuration struct {

2
git/object_scanner.go
View File

@ -42,7 +42,7 @@ type ObjectScanner struct {
// command, they will be returned immediately.
//
// Otherwise, an `*ObjectScanner` is returned with no error.
func NewObjectScanner() (*ObjectScanner, error) {
func NewObjectScanner(osEnv Environment) (*ObjectScanner, error) {
gitdir, err := GitCommonDir()
if err != nil {
return nil, err

20
lfs/gitscanner.go
View File

@ -6,6 +6,7 @@ import (
"sync"
"time"
"github.com/git-lfs/git-lfs/config"
"github.com/git-lfs/git-lfs/filepathfilter"
"github.com/rubyist/tracerx"
)
@ -30,6 +31,7 @@ type GitScanner struct {
closed bool
started time.Time
mu sync.Mutex
cfg *config.Configuration
}
type GitScannerFoundPointer func(*WrappedPointer, error)
@ -41,8 +43,8 @@ type GitScannerSet interface {
// NewGitScanner initializes a *GitScanner for a Git repository in the current
// working directory.
func NewGitScanner(cb GitScannerFoundPointer) *GitScanner {
return &GitScanner{started: time.Now(), FoundPointer: cb}
func NewGitScanner(cfg *config.Configuration, cb GitScannerFoundPointer) *GitScanner {
return &GitScanner{started: time.Now(), FoundPointer: cb, cfg: cfg}
}
// Close stops exits once all processing has stopped, and all resources are
@ -90,7 +92,7 @@ func (s *GitScanner) ScanRangeToRemote(left, right string, cb GitScannerFoundPoi
}
s.mu.Unlock()
return scanLeftRightToChan(s, callback, left, right, s.opts(ScanRangeToRemoteMode))
return scanLeftRightToChan(s, callback, left, right, s.cfg.OSEnv(), s.opts(ScanRangeToRemoteMode))
}
// ScanRefs through all commits reachable by refs contained in "include" and
@ -103,7 +105,7 @@ func (s *GitScanner) ScanRefs(include, exclude []string, cb GitScannerFoundPoint
opts := s.opts(ScanRefsMode)
opts.SkipDeletedBlobs = false
return scanRefsToChan(s, callback, include, exclude, opts)
return scanRefsToChan(s, callback, include, exclude, s.cfg.OSEnv(), opts)
}
// ScanRefRange scans through all commits from the given left and right refs,
@ -116,7 +118,7 @@ func (s *GitScanner) ScanRefRange(left, right string, cb GitScannerFoundPointer)
opts := s.opts(ScanRefsMode)
opts.SkipDeletedBlobs = false
return scanLeftRightToChan(s, callback, left, right, opts)
return scanLeftRightToChan(s, callback, left, right, s.cfg.OSEnv(), opts)
}
// ScanRefWithDeleted scans through all objects in the given ref, including
@ -135,7 +137,7 @@ func (s *GitScanner) ScanRef(ref string, cb GitScannerFoundPointer) error {
opts := s.opts(ScanRefsMode)
opts.SkipDeletedBlobs = true
return scanLeftRightToChan(s, callback, ref, "", opts)
return scanLeftRightToChan(s, callback, ref, "", s.cfg.OSEnv(), opts)
}
// ScanAll scans through all objects in the git repository.
@ -147,7 +149,7 @@ func (s *GitScanner) ScanAll(cb GitScannerFoundPointer) error {
opts := s.opts(ScanAllMode)
opts.SkipDeletedBlobs = false
return scanLeftRightToChan(s, callback, "", "", opts)
return scanLeftRightToChan(s, callback, "", "", s.cfg.OSEnv(), opts)
}
// ScanTree takes a ref and returns WrappedPointer objects in the tree at that
@ -158,7 +160,7 @@ func (s *GitScanner) ScanTree(ref string) error {
if err != nil {
return err
}
return runScanTree(callback, ref, s.Filter)
return runScanTree(callback, ref, s.Filter, s.cfg.OSEnv())
}
// ScanUnpushed scans history for all LFS pointers which have been added but not
@ -189,7 +191,7 @@ func (s *GitScanner) ScanIndex(ref string, cb GitScannerFoundPointer) error {
if err != nil {
return err
}
return scanIndex(callback, ref, s.Filter)
return scanIndex(callback, ref, s.Filter, s.cfg.OSEnv())
}
func (s *GitScanner) opts(mode ScanningMode) *ScanRefsOptions {

9
lfs/gitscanner_catfilebatch.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"github.com/git-lfs/git-lfs/config"
"github.com/git-lfs/git-lfs/git"
)
@ -16,8 +17,8 @@ import (
// pointerCh. If a Git Blob is not an LFS pointer, check the lockableSet to see
// if that blob is for a locked file. Any errors are sent to errCh. An error is
// returned if the 'git cat-file' command fails to start.
func runCatFileBatch(pointerCh chan *WrappedPointer, lockableCh chan string, lockableSet *lockableNameSet, revs *StringChannelWrapper, errCh chan error) error {
scanner, err := NewPointerScanner()
func runCatFileBatch(pointerCh chan *WrappedPointer, lockableCh chan string, lockableSet *lockableNameSet, revs *StringChannelWrapper, errCh chan error, osEnv config.Environment) error {
scanner, err := NewPointerScanner(osEnv)
if err != nil {
return err
}
@ -69,8 +70,8 @@ type PointerScanner struct {
err error
}
func NewPointerScanner() (*PointerScanner, error) {
scanner, err := git.NewObjectScanner()
func NewPointerScanner(osEnv config.Environment) (*PointerScanner, error) {
scanner, err := git.NewObjectScanner(osEnv)
if err != nil {
return nil, err
}

5
lfs/gitscanner_index.go
View File

@ -4,6 +4,7 @@ import (
"strings"
"sync"
"github.com/git-lfs/git-lfs/config"
"github.com/git-lfs/git-lfs/filepathfilter"
)
@ -12,7 +13,7 @@ import (
//
// Ref is the ref at which to scan, which may be "HEAD" if there is at least one
// commit.
func scanIndex(cb GitScannerFoundPointer, ref string, f *filepathfilter.Filter) error {
func scanIndex(cb GitScannerFoundPointer, ref string, f *filepathfilter.Filter, osEnv config.Environment) error {
indexMap := &indexFileMap{
nameMap: make(map[string][]*indexFile),
nameShaPairs: make(map[string]bool),
@ -67,7 +68,7 @@ func scanIndex(cb GitScannerFoundPointer, ref string, f *filepathfilter.Filter)
ch := make(chan gitscannerResult, chanBufSize)
barePointerCh, _, err := catFileBatch(smallShas, nil)
barePointerCh, _, err := catFileBatch(smallShas, nil, osEnv)
if err != nil {
return err
}

9
lfs/gitscanner_refs.go
View File

@ -4,6 +4,7 @@ import (
"encoding/hex"
"regexp"
"github.com/git-lfs/git-lfs/config"
"github.com/git-lfs/git-lfs/git"
)
@ -37,7 +38,7 @@ func noopFoundLockable(name string) {}
// "include" and not reachable by any refs included in "excluded" and returns
// a channel of WrappedPointer objects for all Git LFS pointers it finds.
// Reports unique oids once only, not multiple times if >1 file uses the same content
func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, include, exclude []string, opt *ScanRefsOptions) error {
func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, include, exclude []string, osEnv config.Environment, opt *ScanRefsOptions) error {
if opt == nil {
panic("no scan ref options")
}
@ -64,7 +65,7 @@ func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, inclu
}
}(lockableCb, batchLockableCh)
pointers, checkLockableCh, err := catFileBatch(smallShas, lockableSet)
pointers, checkLockableCh, err := catFileBatch(smallShas, lockableSet, osEnv)
if err != nil {
return err
}
@ -95,8 +96,8 @@ func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, inclu
// scanLeftRightToChan takes a ref and returns a channel of WrappedPointer objects
// for all Git LFS pointers it finds for that ref.
// Reports unique oids once only, not multiple times if >1 file uses the same content
func scanLeftRightToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, refLeft, refRight string, opt *ScanRefsOptions) error {
return scanRefsToChan(scanner, pointerCb, []string{refLeft}, []string{refRight}, opt)
func scanLeftRightToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, refLeft, refRight string, osEnv config.Environment, opt *ScanRefsOptions) error {
return scanRefsToChan(scanner, pointerCb, []string{refLeft}, []string{refRight}, osEnv, opt)
}
// revListShas uses git rev-list to return the list of object sha1s

9
lfs/gitscanner_tree.go
View File

@ -9,6 +9,7 @@ import (
"strconv"
"strings"
"github.com/git-lfs/git-lfs/config"
"github.com/git-lfs/git-lfs/filepathfilter"
"github.com/git-lfs/git-lfs/git"
)
@ -19,7 +20,7 @@ type TreeBlob struct {
Filename string
}
func runScanTree(cb GitScannerFoundPointer, ref string, filter *filepathfilter.Filter) error {
func runScanTree(cb GitScannerFoundPointer, ref string, filter *filepathfilter.Filter, osEnv config.Environment) error {
// We don't use the nameMap approach here since that's imprecise when >1 file
// can be using the same content
treeShas, err := lsTreeBlobs(ref, filter)
@ -27,7 +28,7 @@ func runScanTree(cb GitScannerFoundPointer, ref string, filter *filepathfilter.F
return err
}
pcw, err := catFileBatchTree(treeShas)
pcw, err := catFileBatchTree(treeShas, osEnv)
if err != nil {
return err
}
@ -46,8 +47,8 @@ func runScanTree(cb GitScannerFoundPointer, ref string, filter *filepathfilter.F
// of a git object, given its sha1. The contents will be decoded into
// a Git LFS pointer. treeblobs is a channel over which blob entries
// will be sent. It returns a channel from which point.Pointers can be read.
func catFileBatchTree(treeblobs *TreeBlobChannelWrapper) (*PointerChannelWrapper, error) {
scanner, err := NewPointerScanner()
func catFileBatchTree(treeblobs *TreeBlobChannelWrapper, osEnv config.Environment) (*PointerChannelWrapper, error) {
scanner, err := NewPointerScanner(osEnv)
if err != nil {
return nil, err
}

9
lfs/scanner.go
View File

@ -1,6 +1,9 @@
package lfs
import "github.com/git-lfs/git-lfs/tools"
import (
"github.com/git-lfs/git-lfs/config"
"github.com/git-lfs/git-lfs/tools"
)
const (
// blobSizeCutoff is used to determine which files to scan for Git LFS
@ -45,11 +48,11 @@ func catFileBatchCheck(revs *StringChannelWrapper, lockableSet *lockableNameSet)
// of a git object, given its sha1. The contents will be decoded into
// a Git LFS pointer. revs is a channel over which strings containing Git SHA1s
// will be sent. It returns a channel from which point.Pointers can be read.
func catFileBatch(revs *StringChannelWrapper, lockableSet *lockableNameSet) (*PointerChannelWrapper, chan string, error) {
func catFileBatch(revs *StringChannelWrapper, lockableSet *lockableNameSet, osEnv config.Environment) (*PointerChannelWrapper, chan string, error) {
pointerCh := make(chan *WrappedPointer, chanBufSize)
lockableCh := make(chan string, chanBufSize)
errCh := make(chan error, 5) // shared by 2 goroutines & may add more detail errors?
if err := runCatFileBatch(pointerCh, lockableCh, lockableSet, revs, errCh); err != nil {
if err := runCatFileBatch(pointerCh, lockableCh, lockableSet, revs, errCh, osEnv); err != nil {
return nil, nil, err
}
return NewPointerChannelWrapper(pointerCh, errCh), lockableCh, nil

5
lfs/scanner_git_test.go
View File

@ -10,6 +10,7 @@ import (
"testing"
"time"
"github.com/git-lfs/git-lfs/config"
. "github.com/git-lfs/git-lfs/lfs"
test "github.com/git-lfs/git-lfs/t/cmd/util"
"github.com/stretchr/testify/assert"
@ -91,7 +92,7 @@ func scanUnpushed(remoteName string) ([]*WrappedPointer, error) {
pointers := make([]*WrappedPointer, 0, 10)
var multiErr error
gitscanner := NewGitScanner(func(p *WrappedPointer, err error) {
gitscanner := NewGitScanner(config.New(), func(p *WrappedPointer, err error) {
if err != nil {
if multiErr != nil {
multiErr = fmt.Errorf("%v\n%v", multiErr, err)
@ -193,7 +194,7 @@ func TestScanPreviousVersions(t *testing.T) {
func scanPreviousVersions(t *testing.T, ref string, since time.Time) ([]*WrappedPointer, error) {
pointers := make([]*WrappedPointer, 0, 10)
gitscanner := NewGitScanner(func(p *WrappedPointer, err error) {
gitscanner := NewGitScanner(config.New(), func(p *WrappedPointer, err error) {
if err != nil {
t.Error(err)
return