2016-04-06 19:06:34 +00:00
|
|
|
package commands
|
|
|
|
|
|
|
|
import (
|
|
|
|
"os"
|
|
|
|
|
2016-05-16 14:06:02 +00:00
|
|
|
"github.com/github/git-lfs/errutil"
|
2016-04-06 19:06:34 +00:00
|
|
|
"github.com/github/git-lfs/lfs"
|
2016-07-07 16:16:13 +00:00
|
|
|
"github.com/github/git-lfs/tools"
|
2016-04-06 19:06:34 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
var uploadMissingErr = "%s does not exist in .git/lfs/objects. Tried %s, which matches %s."
|
|
|
|
|
|
|
|
type uploadContext struct {
|
|
|
|
DryRun bool
|
2016-07-07 16:16:13 +00:00
|
|
|
uploadedOids tools.StringSet
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
func newUploadContext(dryRun bool) *uploadContext {
|
2016-04-06 19:06:34 +00:00
|
|
|
return &uploadContext{
|
2016-04-07 16:52:24 +00:00
|
|
|
DryRun: dryRun,
|
2016-07-07 16:16:13 +00:00
|
|
|
uploadedOids: tools.NewStringSet(),
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// AddUpload adds the given oid to the set of oids that have been uploaded in
|
|
|
|
// the current process.
|
2016-04-08 15:59:02 +00:00
|
|
|
func (c *uploadContext) SetUploaded(oid string) {
|
2016-04-07 16:52:24 +00:00
|
|
|
c.uploadedOids.Add(oid)
|
|
|
|
}
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// HasUploaded determines if the given oid has already been uploaded in the
|
|
|
|
// current process.
|
|
|
|
func (c *uploadContext) HasUploaded(oid string) bool {
|
|
|
|
return c.uploadedOids.Contains(oid)
|
|
|
|
}
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
func (c *uploadContext) prepareUpload(unfiltered []*lfs.WrappedPointer) (*lfs.TransferQueue, []*lfs.WrappedPointer) {
|
|
|
|
numUnfiltered := len(unfiltered)
|
|
|
|
uploadables := make([]*lfs.WrappedPointer, 0, numUnfiltered)
|
|
|
|
missingLocalObjects := make([]*lfs.WrappedPointer, 0, numUnfiltered)
|
|
|
|
numObjects := 0
|
|
|
|
totalSize := int64(0)
|
|
|
|
missingSize := int64(0)
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// separate out objects that _should_ be uploaded, but don't exist in
|
|
|
|
// .git/lfs/objects. Those will skipped if the server already has them.
|
|
|
|
for _, p := range unfiltered {
|
|
|
|
// object already uploaded in this process, skip!
|
|
|
|
if c.HasUploaded(p.Oid) {
|
|
|
|
continue
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
numObjects += 1
|
|
|
|
totalSize += p.Size
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
if lfs.ObjectExistsOfSize(p.Oid, p.Size) {
|
|
|
|
uploadables = append(uploadables, p)
|
2016-04-06 19:06:34 +00:00
|
|
|
} else {
|
2016-04-07 16:52:24 +00:00
|
|
|
// We think we need to push this but we don't have it
|
|
|
|
// Store for server checking later
|
|
|
|
missingLocalObjects = append(missingLocalObjects, p)
|
|
|
|
missingSize += p.Size
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// check to see if the server has the missing objects.
|
|
|
|
c.checkMissing(missingLocalObjects, missingSize)
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// build the TransferQueue, automatically skipping any missing objects that
|
|
|
|
// the server already has.
|
|
|
|
uploadQueue := lfs.NewUploadQueue(numObjects, totalSize, c.DryRun)
|
|
|
|
for _, p := range missingLocalObjects {
|
|
|
|
if c.HasUploaded(p.Oid) {
|
|
|
|
uploadQueue.Skip(p.Size)
|
2016-04-06 19:06:34 +00:00
|
|
|
} else {
|
2016-04-07 16:52:24 +00:00
|
|
|
uploadables = append(uploadables, p)
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
return uploadQueue, uploadables
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// This checks the given slice of pointers that don't exist in .git/lfs/objects
|
|
|
|
// against the server. Anything the server already has does not need to be
|
|
|
|
// uploaded again.
|
|
|
|
func (c *uploadContext) checkMissing(missing []*lfs.WrappedPointer, missingSize int64) {
|
|
|
|
numMissing := len(missing)
|
|
|
|
if numMissing == 0 {
|
2016-04-06 19:06:34 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2016-05-25 15:41:47 +00:00
|
|
|
checkQueue := lfs.NewDownloadCheckQueue(numMissing, missingSize)
|
2016-05-26 17:59:13 +00:00
|
|
|
|
|
|
|
// this channel is filled with oids for which Check() succeeded & Transfer() was called
|
|
|
|
transferc := checkQueue.Watch()
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
for _, p := range missing {
|
2016-05-25 15:41:47 +00:00
|
|
|
checkQueue.Add(lfs.NewDownloadable(p))
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
2016-04-07 16:52:24 +00:00
|
|
|
|
2016-04-06 19:06:34 +00:00
|
|
|
done := make(chan int)
|
|
|
|
go func() {
|
|
|
|
for oid := range transferc {
|
2016-04-08 15:59:02 +00:00
|
|
|
c.SetUploaded(oid)
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
done <- 1
|
|
|
|
}()
|
2016-04-07 16:52:24 +00:00
|
|
|
|
2016-04-06 19:06:34 +00:00
|
|
|
// Currently this is needed to flush the batch but is not enough to sync transferc completely
|
|
|
|
checkQueue.Wait()
|
|
|
|
<-done
|
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
func upload(c *uploadContext, unfiltered []*lfs.WrappedPointer) {
|
|
|
|
if c.DryRun {
|
|
|
|
for _, p := range unfiltered {
|
|
|
|
if c.HasUploaded(p.Oid) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
Print("push %s => %s", p.Oid, p.Name)
|
2016-04-08 15:59:02 +00:00
|
|
|
c.SetUploaded(p.Oid)
|
2016-04-07 16:52:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
q, pointers := c.prepareUpload(unfiltered)
|
|
|
|
for _, p := range pointers {
|
|
|
|
u, err := lfs.NewUploadable(p.Oid, p.Name)
|
|
|
|
if err != nil {
|
2016-05-16 14:06:02 +00:00
|
|
|
if errutil.IsCleanPointerError(err) {
|
|
|
|
Exit(uploadMissingErr, p.Oid, p.Name, errutil.ErrorGetContext(err, "pointer").(*lfs.Pointer).Oid)
|
2016-04-07 16:52:24 +00:00
|
|
|
} else {
|
|
|
|
ExitWithError(err)
|
|
|
|
}
|
|
|
|
}
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
q.Add(u)
|
2016-04-08 15:59:02 +00:00
|
|
|
c.SetUploaded(p.Oid)
|
2016-04-07 16:52:24 +00:00
|
|
|
}
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
q.Wait()
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
for _, err := range q.Errors() {
|
2016-05-16 14:06:02 +00:00
|
|
|
if Debugging || errutil.IsFatalError(err) {
|
2016-04-07 16:52:24 +00:00
|
|
|
LoggedError(err, err.Error())
|
|
|
|
} else {
|
2016-08-13 04:20:52 +00:00
|
|
|
var innermsg string = ""
|
2016-05-16 14:06:02 +00:00
|
|
|
if inner := errutil.GetInnerError(err); inner != nil {
|
2016-08-13 04:20:52 +00:00
|
|
|
innermsg = inner.Error()
|
|
|
|
Error(innermsg)
|
|
|
|
}
|
|
|
|
var msg string = err.Error()
|
|
|
|
if msg != innermsg {
|
|
|
|
Error(msg)
|
2016-04-07 16:52:24 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(q.Errors()) > 0 {
|
|
|
|
os.Exit(2)
|
|
|
|
}
|
|
|
|
}
|