2016-04-06 19:06:34 +00:00
|
|
|
package commands
|
|
|
|
|
|
|
|
import (
|
|
|
|
"os"
|
|
|
|
|
2016-11-15 17:01:18 +00:00
|
|
|
"github.com/git-lfs/git-lfs/errors"
|
|
|
|
"github.com/git-lfs/git-lfs/lfs"
|
|
|
|
"github.com/git-lfs/git-lfs/tools"
|
2016-04-06 19:06:34 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
var uploadMissingErr = "%s does not exist in .git/lfs/objects. Tried %s, which matches %s."
|
|
|
|
|
|
|
|
type uploadContext struct {
|
|
|
|
DryRun bool
|
2016-07-07 16:16:13 +00:00
|
|
|
uploadedOids tools.StringSet
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
func newUploadContext(dryRun bool) *uploadContext {
|
2016-04-06 19:06:34 +00:00
|
|
|
return &uploadContext{
|
2016-04-07 16:52:24 +00:00
|
|
|
DryRun: dryRun,
|
2016-07-07 16:16:13 +00:00
|
|
|
uploadedOids: tools.NewStringSet(),
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// AddUpload adds the given oid to the set of oids that have been uploaded in
|
|
|
|
// the current process.
|
2016-04-08 15:59:02 +00:00
|
|
|
func (c *uploadContext) SetUploaded(oid string) {
|
2016-04-07 16:52:24 +00:00
|
|
|
c.uploadedOids.Add(oid)
|
|
|
|
}
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// HasUploaded determines if the given oid has already been uploaded in the
|
|
|
|
// current process.
|
|
|
|
func (c *uploadContext) HasUploaded(oid string) bool {
|
|
|
|
return c.uploadedOids.Contains(oid)
|
|
|
|
}
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
func (c *uploadContext) prepareUpload(unfiltered []*lfs.WrappedPointer) (*lfs.TransferQueue, []*lfs.WrappedPointer) {
|
|
|
|
numUnfiltered := len(unfiltered)
|
|
|
|
uploadables := make([]*lfs.WrappedPointer, 0, numUnfiltered)
|
|
|
|
missingLocalObjects := make([]*lfs.WrappedPointer, 0, numUnfiltered)
|
|
|
|
missingSize := int64(0)
|
2016-12-07 02:59:42 +00:00
|
|
|
meter := buildProgressMeter()
|
2016-12-07 03:29:18 +00:00
|
|
|
meter.DryRun = c.DryRun
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-10-20 18:07:40 +00:00
|
|
|
// XXX(taylor): temporary measure to fix duplicate (broken) results from
|
|
|
|
// scanner
|
|
|
|
uniqOids := tools.NewStringSet()
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// separate out objects that _should_ be uploaded, but don't exist in
|
|
|
|
// .git/lfs/objects. Those will skipped if the server already has them.
|
|
|
|
for _, p := range unfiltered {
|
2016-10-20 18:07:40 +00:00
|
|
|
// object already uploaded in this process, or we've already
|
|
|
|
// seen this OID (see above), skip!
|
|
|
|
if uniqOids.Contains(p.Oid) || c.HasUploaded(p.Oid) {
|
2016-04-07 16:52:24 +00:00
|
|
|
continue
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
2016-10-20 18:07:40 +00:00
|
|
|
uniqOids.Add(p.Oid)
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-12-07 02:59:42 +00:00
|
|
|
// estimate in meter early (even if it's not going into uploadables), since
|
|
|
|
// we will call Skip() based on the results of the download check queue.
|
|
|
|
meter.AddEstimate(p.Size)
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
if lfs.ObjectExistsOfSize(p.Oid, p.Size) {
|
|
|
|
uploadables = append(uploadables, p)
|
2016-04-06 19:06:34 +00:00
|
|
|
} else {
|
2016-04-07 16:52:24 +00:00
|
|
|
// We think we need to push this but we don't have it
|
|
|
|
// Store for server checking later
|
|
|
|
missingLocalObjects = append(missingLocalObjects, p)
|
|
|
|
missingSize += p.Size
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// check to see if the server has the missing objects.
|
|
|
|
c.checkMissing(missingLocalObjects, missingSize)
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// build the TransferQueue, automatically skipping any missing objects that
|
|
|
|
// the server already has.
|
2016-12-07 03:41:35 +00:00
|
|
|
uploadQueue := lfs.NewUploadQueue(lfs.WithProgress(meter), lfs.DryRun(c.DryRun))
|
2016-04-07 16:52:24 +00:00
|
|
|
for _, p := range missingLocalObjects {
|
|
|
|
if c.HasUploaded(p.Oid) {
|
2016-10-20 18:07:40 +00:00
|
|
|
// if the server already has this object, call Skip() on
|
|
|
|
// the progressmeter to decrement the number of files by
|
|
|
|
// 1 and the number of bytes by `p.Size`.
|
2016-04-07 16:52:24 +00:00
|
|
|
uploadQueue.Skip(p.Size)
|
2016-04-06 19:06:34 +00:00
|
|
|
} else {
|
2016-04-07 16:52:24 +00:00
|
|
|
uploadables = append(uploadables, p)
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
return uploadQueue, uploadables
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
// This checks the given slice of pointers that don't exist in .git/lfs/objects
|
|
|
|
// against the server. Anything the server already has does not need to be
|
|
|
|
// uploaded again.
|
|
|
|
func (c *uploadContext) checkMissing(missing []*lfs.WrappedPointer, missingSize int64) {
|
|
|
|
numMissing := len(missing)
|
|
|
|
if numMissing == 0 {
|
2016-04-06 19:06:34 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2016-12-07 03:05:27 +00:00
|
|
|
checkQueue := lfs.NewDownloadCheckQueue()
|
2016-11-17 17:05:56 +00:00
|
|
|
transferCh := checkQueue.Watch()
|
2016-04-07 16:52:24 +00:00
|
|
|
|
2016-04-06 19:06:34 +00:00
|
|
|
done := make(chan int)
|
|
|
|
go func() {
|
2016-11-17 17:05:56 +00:00
|
|
|
// this channel is filled with oids for which Check() succeeded
|
|
|
|
// and Transfer() was called
|
2016-11-17 15:08:18 +00:00
|
|
|
for oid := range transferCh {
|
2016-04-08 15:59:02 +00:00
|
|
|
c.SetUploaded(oid)
|
2016-04-06 19:06:34 +00:00
|
|
|
}
|
|
|
|
done <- 1
|
|
|
|
}()
|
2016-04-07 16:52:24 +00:00
|
|
|
|
2016-11-17 15:08:18 +00:00
|
|
|
for _, p := range missing {
|
|
|
|
checkQueue.Add(lfs.NewDownloadable(p))
|
|
|
|
}
|
|
|
|
|
2016-11-17 17:05:56 +00:00
|
|
|
// Currently this is needed to flush the batch but is not enough to sync
|
|
|
|
// transferc completely. By the time that checkQueue.Wait() returns, the
|
|
|
|
// transferCh will have been closed, allowing the goroutine above to
|
|
|
|
// send "1" into the `done` channel.
|
2016-04-06 19:06:34 +00:00
|
|
|
checkQueue.Wait()
|
|
|
|
<-done
|
|
|
|
}
|
|
|
|
|
2016-11-16 20:25:05 +00:00
|
|
|
func uploadPointers(c *uploadContext, unfiltered []*lfs.WrappedPointer) {
|
2016-04-07 16:52:24 +00:00
|
|
|
if c.DryRun {
|
|
|
|
for _, p := range unfiltered {
|
|
|
|
if c.HasUploaded(p.Oid) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
Print("push %s => %s", p.Oid, p.Name)
|
2016-04-08 15:59:02 +00:00
|
|
|
c.SetUploaded(p.Oid)
|
2016-04-07 16:52:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
q, pointers := c.prepareUpload(unfiltered)
|
|
|
|
for _, p := range pointers {
|
|
|
|
u, err := lfs.NewUploadable(p.Oid, p.Name)
|
|
|
|
if err != nil {
|
2016-08-18 20:20:33 +00:00
|
|
|
if errors.IsCleanPointerError(err) {
|
2016-08-19 20:03:39 +00:00
|
|
|
Exit(uploadMissingErr, p.Oid, p.Name, errors.GetContext(err, "pointer").(*lfs.Pointer).Oid)
|
2016-04-07 16:52:24 +00:00
|
|
|
} else {
|
|
|
|
ExitWithError(err)
|
|
|
|
}
|
|
|
|
}
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
q.Add(u)
|
2016-04-08 15:59:02 +00:00
|
|
|
c.SetUploaded(p.Oid)
|
2016-04-07 16:52:24 +00:00
|
|
|
}
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
q.Wait()
|
2016-04-06 19:06:34 +00:00
|
|
|
|
2016-04-07 16:52:24 +00:00
|
|
|
for _, err := range q.Errors() {
|
2016-08-16 18:03:37 +00:00
|
|
|
FullError(err)
|
2016-04-07 16:52:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if len(q.Errors()) > 0 {
|
|
|
|
os.Exit(2)
|
|
|
|
}
|
|
|
|
}
|