Fixes a race condition accessing the maps inside goroutines
This commit is contained in:
parent
ca4b6d30e2
commit
c3bc64b15f
@ -101,7 +101,9 @@ func init() {
|
|||||||
|
|
||||||
func pointersToFetchForRef(ref string) ([]*lfs.WrappedPointer, error) {
|
func pointersToFetchForRef(ref string) ([]*lfs.WrappedPointer, error) {
|
||||||
// Use SkipDeletedBlobs to avoid fetching ALL previous versions of modified files
|
// Use SkipDeletedBlobs to avoid fetching ALL previous versions of modified files
|
||||||
opts := &lfs.ScanRefsOptions{ScanMode: lfs.ScanRefsMode, SkipDeletedBlobs: true}
|
opts := lfs.NewScanRefsOptions()
|
||||||
|
opts.ScanMode = lfs.ScanRefsMode
|
||||||
|
opts.SkipDeletedBlobs = true
|
||||||
return lfs.ScanRefs(ref, "", opts)
|
return lfs.ScanRefs(ref, "", opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -200,7 +202,9 @@ func fetchAll() bool {
|
|||||||
func scanAll() []*lfs.WrappedPointer {
|
func scanAll() []*lfs.WrappedPointer {
|
||||||
// converts to `git rev-list --all`
|
// converts to `git rev-list --all`
|
||||||
// We only pick up objects in real commits and not the reflog
|
// We only pick up objects in real commits and not the reflog
|
||||||
opts := &lfs.ScanRefsOptions{ScanMode: lfs.ScanAllMode, SkipDeletedBlobs: false}
|
opts := lfs.NewScanRefsOptions()
|
||||||
|
opts.ScanMode = lfs.ScanAllMode
|
||||||
|
opts.SkipDeletedBlobs = false
|
||||||
|
|
||||||
// This could be a long process so use the chan version & report progress
|
// This could be a long process so use the chan version & report progress
|
||||||
Print("Scanning for all objects ever referenced...")
|
Print("Scanning for all objects ever referenced...")
|
||||||
|
@ -71,7 +71,10 @@ func prePushCommand(cmd *cobra.Command, args []string) {
|
|||||||
|
|
||||||
func prePushRef(left, right string) {
|
func prePushRef(left, right string) {
|
||||||
// Just use scanner here
|
// Just use scanner here
|
||||||
scanOpt := &lfs.ScanRefsOptions{ScanMode: lfs.ScanLeftToRemoteMode, RemoteName: lfs.Config.CurrentRemote}
|
scanOpt := lfs.NewScanRefsOptions()
|
||||||
|
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
|
||||||
|
scanOpt.RemoteName = lfs.Config.CurrentRemote
|
||||||
|
|
||||||
pointers, err := lfs.ScanRefs(left, right, scanOpt)
|
pointers, err := lfs.ScanRefs(left, right, scanOpt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Panic(err, "Error scanning for Git LFS files")
|
Panic(err, "Error scanning for Git LFS files")
|
||||||
|
@ -37,7 +37,9 @@ func uploadsBetweenRefs(left string, right string) *lfs.TransferQueue {
|
|||||||
func uploadsBetweenRefAndRemote(remote string, refs []string) *lfs.TransferQueue {
|
func uploadsBetweenRefAndRemote(remote string, refs []string) *lfs.TransferQueue {
|
||||||
tracerx.Printf("Upload refs %v to remote %v", remote, refs)
|
tracerx.Printf("Upload refs %v to remote %v", remote, refs)
|
||||||
|
|
||||||
scanOpt := &lfs.ScanRefsOptions{ScanMode: lfs.ScanLeftToRemoteMode, RemoteName: remote}
|
scanOpt := lfs.NewScanRefsOptions()
|
||||||
|
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
|
||||||
|
scanOpt.RemoteName = remote
|
||||||
|
|
||||||
if pushAll {
|
if pushAll {
|
||||||
if len(refs) == 0 {
|
if len(refs) == 0 {
|
||||||
|
@ -10,6 +10,7 @@ import (
|
|||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/github/git-lfs/git"
|
"github.com/github/git-lfs/git"
|
||||||
@ -76,6 +77,27 @@ type ScanRefsOptions struct {
|
|||||||
RemoteName string
|
RemoteName string
|
||||||
SkipDeletedBlobs bool
|
SkipDeletedBlobs bool
|
||||||
nameMap map[string]string
|
nameMap map[string]string
|
||||||
|
mutex *sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *ScanRefsOptions) GetName(sha string) (string, bool) {
|
||||||
|
o.mutex.Lock()
|
||||||
|
name, ok := o.nameMap[sha]
|
||||||
|
o.mutex.Unlock()
|
||||||
|
return name, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *ScanRefsOptions) SetName(sha, name string) {
|
||||||
|
o.mutex.Lock()
|
||||||
|
o.nameMap[sha] = name
|
||||||
|
o.mutex.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewScanRefsOptions() *ScanRefsOptions {
|
||||||
|
return &ScanRefsOptions{
|
||||||
|
nameMap: make(map[string]string, 0),
|
||||||
|
mutex: &sync.Mutex{},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ScanRefs takes a ref and returns a slice of WrappedPointer objects
|
// ScanRefs takes a ref and returns a slice of WrappedPointer objects
|
||||||
@ -100,19 +122,18 @@ func ScanRefs(refLeft, refRight string, opt *ScanRefsOptions) ([]*WrappedPointer
|
|||||||
// Reports unique oids once only, not multiple times if >1 file uses the same content
|
// Reports unique oids once only, not multiple times if >1 file uses the same content
|
||||||
func ScanRefsToChan(refLeft, refRight string, opt *ScanRefsOptions) (<-chan *WrappedPointer, error) {
|
func ScanRefsToChan(refLeft, refRight string, opt *ScanRefsOptions) (<-chan *WrappedPointer, error) {
|
||||||
if opt == nil {
|
if opt == nil {
|
||||||
opt = &ScanRefsOptions{}
|
opt = NewScanRefsOptions()
|
||||||
}
|
}
|
||||||
if refLeft == "" {
|
if refLeft == "" {
|
||||||
opt.ScanMode = ScanAllMode
|
opt.ScanMode = ScanAllMode
|
||||||
}
|
}
|
||||||
opt.nameMap = make(map[string]string, 0)
|
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
defer func() {
|
defer func() {
|
||||||
tracerx.PerformanceSince("scan", start)
|
tracerx.PerformanceSince("scan", start)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
revs, err := revListShas(refLeft, refRight, *opt)
|
revs, err := revListShas(refLeft, refRight, opt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -130,7 +151,7 @@ func ScanRefsToChan(refLeft, refRight string, opt *ScanRefsOptions) (<-chan *Wra
|
|||||||
retchan := make(chan *WrappedPointer, chanBufSize)
|
retchan := make(chan *WrappedPointer, chanBufSize)
|
||||||
go func() {
|
go func() {
|
||||||
for p := range pointerc {
|
for p := range pointerc {
|
||||||
if name, ok := opt.nameMap[p.Sha1]; ok {
|
if name, ok := opt.GetName(p.Sha1); ok {
|
||||||
p.Name = name
|
p.Name = name
|
||||||
}
|
}
|
||||||
retchan <- p
|
retchan <- p
|
||||||
@ -141,23 +162,44 @@ func ScanRefsToChan(refLeft, refRight string, opt *ScanRefsOptions) (<-chan *Wra
|
|||||||
return retchan, nil
|
return retchan, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type indexFileMap struct {
|
||||||
|
nameMap map[string]*indexFile
|
||||||
|
mutex *sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *indexFileMap) Get(sha string) (*indexFile, bool) {
|
||||||
|
m.mutex.Lock()
|
||||||
|
index, ok := m.nameMap[sha]
|
||||||
|
m.mutex.Unlock()
|
||||||
|
return index, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *indexFileMap) Set(sha string, index *indexFile) {
|
||||||
|
m.mutex.Lock()
|
||||||
|
m.nameMap[sha] = index
|
||||||
|
m.mutex.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
// ScanIndex returns a slice of WrappedPointer objects for all
|
// ScanIndex returns a slice of WrappedPointer objects for all
|
||||||
// Git LFS pointers it finds in the index.
|
// Git LFS pointers it finds in the index.
|
||||||
// Reports unique oids once only, not multiple times if >1 file uses the same content
|
// Reports unique oids once only, not multiple times if >1 file uses the same content
|
||||||
func ScanIndex() ([]*WrappedPointer, error) {
|
func ScanIndex() ([]*WrappedPointer, error) {
|
||||||
nameMap := make(map[string]*indexFile, 0)
|
indexMap := &indexFileMap{
|
||||||
|
nameMap: make(map[string]*indexFile, 0),
|
||||||
|
mutex: &sync.Mutex{},
|
||||||
|
}
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
defer func() {
|
defer func() {
|
||||||
tracerx.PerformanceSince("scan-staging", start)
|
tracerx.PerformanceSince("scan-staging", start)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
revs, err := revListIndex(false, nameMap)
|
revs, err := revListIndex(false, indexMap)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
cachedRevs, err := revListIndex(true, nameMap)
|
cachedRevs, err := revListIndex(true, indexMap)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -191,7 +233,7 @@ func ScanIndex() ([]*WrappedPointer, error) {
|
|||||||
|
|
||||||
pointers := make([]*WrappedPointer, 0)
|
pointers := make([]*WrappedPointer, 0)
|
||||||
for p := range pointerc {
|
for p := range pointerc {
|
||||||
if e, ok := nameMap[p.Sha1]; ok {
|
if e, ok := indexMap.Get(p.Sha1); ok {
|
||||||
p.Name = e.Name
|
p.Name = e.Name
|
||||||
p.Status = e.Status
|
p.Status = e.Status
|
||||||
p.SrcName = e.SrcName
|
p.SrcName = e.SrcName
|
||||||
@ -206,7 +248,7 @@ func ScanIndex() ([]*WrappedPointer, error) {
|
|||||||
// revListShas uses git rev-list to return the list of object sha1s
|
// revListShas uses git rev-list to return the list of object sha1s
|
||||||
// for the given ref. If all is true, ref is ignored. It returns a
|
// for the given ref. If all is true, ref is ignored. It returns a
|
||||||
// channel from which sha1 strings can be read.
|
// channel from which sha1 strings can be read.
|
||||||
func revListShas(refLeft, refRight string, opt ScanRefsOptions) (chan string, error) {
|
func revListShas(refLeft, refRight string, opt *ScanRefsOptions) (chan string, error) {
|
||||||
refArgs := []string{"rev-list", "--objects"}
|
refArgs := []string{"rev-list", "--objects"}
|
||||||
switch opt.ScanMode {
|
switch opt.ScanMode {
|
||||||
case ScanRefsMode:
|
case ScanRefsMode:
|
||||||
@ -247,7 +289,7 @@ func revListShas(refLeft, refRight string, opt ScanRefsOptions) (chan string, er
|
|||||||
|
|
||||||
sha1 := line[0:40]
|
sha1 := line[0:40]
|
||||||
if len(line) > 40 {
|
if len(line) > 40 {
|
||||||
opt.nameMap[sha1] = line[41:len(line)]
|
opt.SetName(sha1, line[41:len(line)])
|
||||||
}
|
}
|
||||||
revs <- sha1
|
revs <- sha1
|
||||||
}
|
}
|
||||||
@ -260,7 +302,7 @@ func revListShas(refLeft, refRight string, opt ScanRefsOptions) (chan string, er
|
|||||||
// revListIndex uses git diff-index to return the list of object sha1s
|
// revListIndex uses git diff-index to return the list of object sha1s
|
||||||
// for in the indexf. It returns a channel from which sha1 strings can be read.
|
// for in the indexf. It returns a channel from which sha1 strings can be read.
|
||||||
// The namMap will be filled indexFile pointers mapping sha1s to indexFiles.
|
// The namMap will be filled indexFile pointers mapping sha1s to indexFiles.
|
||||||
func revListIndex(cache bool, nameMap map[string]*indexFile) (chan string, error) {
|
func revListIndex(cache bool, indexMap *indexFileMap) (chan string, error) {
|
||||||
cmdArgs := []string{"diff-index", "-M"}
|
cmdArgs := []string{"diff-index", "-M"}
|
||||||
if cache {
|
if cache {
|
||||||
cmdArgs = append(cmdArgs, "--cached")
|
cmdArgs = append(cmdArgs, "--cached")
|
||||||
@ -297,7 +339,7 @@ func revListIndex(cache bool, nameMap map[string]*indexFile) (chan string, error
|
|||||||
if status == "M" {
|
if status == "M" {
|
||||||
sha1 = description[2] // This one is modified but not added
|
sha1 = description[2] // This one is modified but not added
|
||||||
}
|
}
|
||||||
nameMap[sha1] = &indexFile{files[len(files)-1], files[0], status}
|
indexMap.Set(sha1, &indexFile{files[len(files)-1], files[0], status})
|
||||||
revs <- sha1
|
revs <- sha1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user