2017-11-29 19:07:44 +00:00
|
|
|
package tq
|
2015-07-10 19:25:59 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
2018-03-07 00:35:50 +00:00
|
|
|
"math"
|
2015-07-10 20:05:04 +00:00
|
|
|
"os"
|
2016-12-07 16:27:51 +00:00
|
|
|
"path/filepath"
|
2018-01-06 02:01:50 +00:00
|
|
|
"strings"
|
2016-02-23 19:00:45 +00:00
|
|
|
"sync"
|
2015-07-10 20:05:04 +00:00
|
|
|
"sync/atomic"
|
2015-07-27 16:43:41 +00:00
|
|
|
"time"
|
2015-07-10 19:25:59 +00:00
|
|
|
|
2017-11-22 22:07:24 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tasklog"
|
2017-11-22 02:25:54 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tools"
|
2017-11-16 16:35:44 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tools/humanize"
|
2015-07-10 19:25:59 +00:00
|
|
|
)
|
|
|
|
|
2017-11-29 18:52:22 +00:00
|
|
|
// Meter provides a progress bar type output for the TransferQueue. It
|
2015-07-27 20:55:23 +00:00
|
|
|
// is given an estimated file count and size up front and tracks the number of
|
|
|
|
// files and bytes transferred as well as the number of files and bytes that
|
|
|
|
// get skipped because the transfer is unnecessary.
|
2017-11-29 18:52:22 +00:00
|
|
|
type Meter struct {
|
2015-07-27 20:41:57 +00:00
|
|
|
finishedFiles int64 // int64s must come first for struct alignment
|
|
|
|
transferringFiles int64
|
|
|
|
estimatedBytes int64
|
2018-01-04 22:34:07 +00:00
|
|
|
lastBytes int64
|
2015-07-27 16:43:41 +00:00
|
|
|
currentBytes int64
|
2018-01-04 22:34:07 +00:00
|
|
|
sampleCount uint64
|
|
|
|
avgBytes float64
|
|
|
|
lastAvg time.Time
|
2016-06-08 09:35:02 +00:00
|
|
|
estimatedFiles int32
|
2017-11-16 16:35:44 +00:00
|
|
|
paused uint32
|
2015-07-27 16:43:41 +00:00
|
|
|
fileIndex map[string]int64 // Maps a file name to its transfer number
|
2016-02-23 19:00:45 +00:00
|
|
|
fileIndexMutex *sync.Mutex
|
2017-11-22 22:07:24 +00:00
|
|
|
updates chan *tasklog.Update
|
2018-02-24 17:36:01 +00:00
|
|
|
|
|
|
|
DryRun bool
|
|
|
|
Logger *tools.SyncWriter
|
|
|
|
Direction Direction
|
2015-07-10 19:25:59 +00:00
|
|
|
}
|
|
|
|
|
2016-12-07 16:27:51 +00:00
|
|
|
type env interface {
|
|
|
|
Get(key string) (val string, ok bool)
|
|
|
|
}
|
|
|
|
|
2017-12-07 22:13:11 +00:00
|
|
|
func (m *Meter) LoggerFromEnv(os env) *tools.SyncWriter {
|
|
|
|
name, _ := os.Get("GIT_LFS_PROGRESS")
|
|
|
|
if len(name) < 1 {
|
|
|
|
return nil
|
2017-11-29 19:00:45 +00:00
|
|
|
}
|
2017-12-07 22:13:11 +00:00
|
|
|
return m.LoggerToFile(name)
|
2017-11-29 19:00:45 +00:00
|
|
|
}
|
|
|
|
|
2017-12-07 22:13:11 +00:00
|
|
|
func (m *Meter) LoggerToFile(name string) *tools.SyncWriter {
|
2017-11-29 19:00:45 +00:00
|
|
|
printErr := func(err string) {
|
|
|
|
fmt.Fprintf(os.Stderr, "Error creating progress logger: %s\n", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if !filepath.IsAbs(name) {
|
|
|
|
printErr("GIT_LFS_PROGRESS must be an absolute path")
|
2017-12-07 22:13:11 +00:00
|
|
|
return nil
|
2017-11-29 19:00:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil {
|
|
|
|
printErr(err.Error())
|
2017-12-07 22:13:11 +00:00
|
|
|
return nil
|
2017-11-29 19:00:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
file, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
|
|
|
|
if err != nil {
|
|
|
|
printErr(err.Error())
|
2017-12-07 22:13:11 +00:00
|
|
|
return nil
|
2017-11-29 19:00:45 +00:00
|
|
|
}
|
|
|
|
|
2017-12-07 22:13:11 +00:00
|
|
|
return tools.NewSyncWriter(file)
|
2017-11-29 19:00:45 +00:00
|
|
|
}
|
|
|
|
|
2017-11-29 18:52:22 +00:00
|
|
|
// NewMeter creates a new Meter.
|
2017-12-07 22:13:11 +00:00
|
|
|
func NewMeter() *Meter {
|
2017-11-29 18:52:22 +00:00
|
|
|
m := &Meter{
|
2015-07-27 20:41:57 +00:00
|
|
|
fileIndex: make(map[string]int64),
|
2016-02-23 19:00:45 +00:00
|
|
|
fileIndexMutex: &sync.Mutex{},
|
2017-11-22 22:07:24 +00:00
|
|
|
updates: make(chan *tasklog.Update),
|
2015-07-10 19:25:59 +00:00
|
|
|
}
|
2016-12-07 16:27:51 +00:00
|
|
|
|
|
|
|
return m
|
2015-07-31 14:30:08 +00:00
|
|
|
}
|
2015-07-27 16:43:41 +00:00
|
|
|
|
2016-12-07 23:54:24 +00:00
|
|
|
// Start begins sending status updates to the optional log file, and stdout.
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) Start() {
|
|
|
|
if m == nil {
|
2017-11-29 18:48:08 +00:00
|
|
|
return
|
|
|
|
}
|
2017-12-07 21:44:19 +00:00
|
|
|
atomic.StoreUint32(&m.paused, 0)
|
2015-07-27 16:43:41 +00:00
|
|
|
}
|
|
|
|
|
2017-01-26 16:31:12 +00:00
|
|
|
// Pause stops sending status updates temporarily, until Start() is called again.
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) Pause() {
|
|
|
|
if m == nil {
|
2017-11-29 18:48:08 +00:00
|
|
|
return
|
|
|
|
}
|
2017-12-07 21:44:19 +00:00
|
|
|
atomic.StoreUint32(&m.paused, 1)
|
2017-01-24 21:11:38 +00:00
|
|
|
}
|
|
|
|
|
2016-12-07 23:42:50 +00:00
|
|
|
// Add tells the progress meter that a single file of the given size will
|
|
|
|
// possibly be transferred. If a file doesn't need to be transferred for some
|
|
|
|
// reason, be sure to call Skip(int64) with the same size.
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) Add(size int64) {
|
|
|
|
if m == nil {
|
2017-11-29 18:48:08 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-01-05 01:10:00 +00:00
|
|
|
defer m.update(false)
|
2017-12-07 21:44:19 +00:00
|
|
|
atomic.AddInt32(&m.estimatedFiles, 1)
|
|
|
|
atomic.AddInt64(&m.estimatedBytes, size)
|
2016-12-07 23:42:50 +00:00
|
|
|
}
|
|
|
|
|
2015-07-27 20:55:23 +00:00
|
|
|
// Skip tells the progress meter that a file of size `size` is being skipped
|
|
|
|
// because the transfer is unnecessary.
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) Skip(size int64) {
|
|
|
|
if m == nil {
|
2017-11-29 18:48:08 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-01-05 01:10:00 +00:00
|
|
|
defer m.update(false)
|
2018-01-06 02:01:50 +00:00
|
|
|
atomic.AddInt64(&m.finishedFiles, 1)
|
|
|
|
atomic.AddInt64(&m.currentBytes, size)
|
2016-12-07 02:27:16 +00:00
|
|
|
}
|
2016-06-08 09:35:02 +00:00
|
|
|
|
2016-12-07 02:27:16 +00:00
|
|
|
// StartTransfer tells the progress meter that a transferring file is being
|
|
|
|
// added to the TransferQueue.
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) StartTransfer(name string) {
|
|
|
|
if m == nil {
|
2017-11-29 18:48:08 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-01-05 01:10:00 +00:00
|
|
|
defer m.update(false)
|
2017-12-07 21:44:19 +00:00
|
|
|
idx := atomic.AddInt64(&m.transferringFiles, 1)
|
|
|
|
m.fileIndexMutex.Lock()
|
|
|
|
m.fileIndex[name] = idx
|
|
|
|
m.fileIndexMutex.Unlock()
|
2015-07-10 19:25:59 +00:00
|
|
|
}
|
|
|
|
|
2015-07-27 20:55:23 +00:00
|
|
|
// TransferBytes increments the number of bytes transferred
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) TransferBytes(direction, name string, read, total int64, current int) {
|
|
|
|
if m == nil {
|
2017-11-29 18:48:08 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-01-05 01:10:00 +00:00
|
|
|
defer m.update(false)
|
2018-01-04 22:34:07 +00:00
|
|
|
|
2018-01-17 22:32:16 +00:00
|
|
|
now := time.Now()
|
2018-01-04 22:34:07 +00:00
|
|
|
since := now.Sub(m.lastAvg)
|
2017-12-07 21:44:19 +00:00
|
|
|
atomic.AddInt64(&m.currentBytes, int64(current))
|
2018-01-04 22:34:07 +00:00
|
|
|
atomic.AddInt64(&m.lastBytes, int64(current))
|
|
|
|
|
|
|
|
if since > time.Second {
|
|
|
|
m.lastAvg = now
|
|
|
|
|
|
|
|
bps := float64(m.lastBytes) / since.Seconds()
|
|
|
|
|
|
|
|
m.avgBytes = (m.avgBytes*float64(m.sampleCount) + bps) / (float64(m.sampleCount) + 1.0)
|
|
|
|
|
|
|
|
atomic.StoreInt64(&m.lastBytes, 0)
|
|
|
|
atomic.AddUint64(&m.sampleCount, 1)
|
|
|
|
}
|
|
|
|
|
2017-12-07 21:44:19 +00:00
|
|
|
m.logBytes(direction, name, read, total)
|
2015-07-27 20:55:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// FinishTransfer increments the finished transfer count
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) FinishTransfer(name string) {
|
|
|
|
if m == nil {
|
2017-11-29 18:48:08 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-01-05 01:10:00 +00:00
|
|
|
defer m.update(false)
|
2017-12-07 21:44:19 +00:00
|
|
|
atomic.AddInt64(&m.finishedFiles, 1)
|
|
|
|
m.fileIndexMutex.Lock()
|
|
|
|
delete(m.fileIndex, name)
|
|
|
|
m.fileIndexMutex.Unlock()
|
2015-07-10 19:25:59 +00:00
|
|
|
}
|
|
|
|
|
2018-01-04 18:49:41 +00:00
|
|
|
// Flush sends the latest progress update, while leaving the meter active.
|
|
|
|
func (m *Meter) Flush() {
|
2018-01-05 01:10:00 +00:00
|
|
|
if m == nil {
|
2018-01-04 18:49:41 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-01-05 01:10:00 +00:00
|
|
|
m.update(true)
|
2018-01-04 18:49:41 +00:00
|
|
|
}
|
|
|
|
|
2017-11-29 18:52:22 +00:00
|
|
|
// Finish shuts down the Meter.
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) Finish() {
|
|
|
|
if m == nil {
|
2017-11-29 18:48:08 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-01-05 01:10:00 +00:00
|
|
|
m.update(false)
|
2017-12-07 21:44:19 +00:00
|
|
|
close(m.updates)
|
2015-07-27 16:43:41 +00:00
|
|
|
}
|
|
|
|
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) Updates() <-chan *tasklog.Update {
|
|
|
|
if m == nil {
|
2017-11-29 18:48:08 +00:00
|
|
|
return nil
|
|
|
|
}
|
2017-12-07 21:44:19 +00:00
|
|
|
return m.updates
|
2015-07-10 20:05:04 +00:00
|
|
|
}
|
|
|
|
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) Throttled() bool {
|
2017-11-16 16:35:44 +00:00
|
|
|
return true
|
2015-07-27 16:43:41 +00:00
|
|
|
}
|
|
|
|
|
2018-01-05 01:10:00 +00:00
|
|
|
func (m *Meter) update(force bool) {
|
2017-12-07 21:44:19 +00:00
|
|
|
if m.skipUpdate() {
|
2015-07-27 16:43:41 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-12-07 21:44:19 +00:00
|
|
|
m.updates <- &tasklog.Update{
|
2018-01-05 01:10:00 +00:00
|
|
|
S: m.str(),
|
|
|
|
At: time.Now(),
|
|
|
|
Force: force,
|
2017-11-16 16:35:44 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) skipUpdate() bool {
|
2017-12-07 22:13:11 +00:00
|
|
|
return m.DryRun ||
|
2018-01-06 02:01:50 +00:00
|
|
|
m.estimatedFiles == 0 ||
|
2017-12-07 21:44:19 +00:00
|
|
|
atomic.LoadUint32(&m.paused) == 1
|
2017-11-16 16:35:44 +00:00
|
|
|
}
|
|
|
|
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) str() string {
|
2018-01-06 02:01:50 +00:00
|
|
|
// (Uploading|Downloading) LFS objects: 100% (10/10) 100 MiB | 10 MiB/s
|
|
|
|
|
|
|
|
direction := strings.Title(m.Direction.String()) + "ing"
|
|
|
|
percentage := 100 * float64(m.finishedFiles) / float64(m.estimatedFiles)
|
2016-12-27 19:17:21 +00:00
|
|
|
|
2018-01-06 02:01:50 +00:00
|
|
|
return fmt.Sprintf("%s LFS objects: %3.f%% (%d/%d), %s | %s",
|
|
|
|
direction,
|
|
|
|
percentage,
|
|
|
|
m.finishedFiles, m.estimatedFiles,
|
2018-03-07 00:35:50 +00:00
|
|
|
humanize.FormatBytes(clamp(m.currentBytes)),
|
|
|
|
humanize.FormatByteRate(clampf(m.avgBytes), time.Second))
|
|
|
|
}
|
|
|
|
|
|
|
|
// clamp clamps the given "x" within the acceptable domain of the uint64 integer
|
|
|
|
// type, so as to prevent over- and underflow.
|
|
|
|
func clamp(x int64) uint64 {
|
|
|
|
if x < 0 {
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
if x > math.MaxInt64 {
|
|
|
|
return math.MaxUint64
|
|
|
|
}
|
|
|
|
return uint64(x)
|
|
|
|
}
|
|
|
|
|
|
|
|
func clampf(x float64) uint64 {
|
|
|
|
if x < 0 {
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
if x > math.MaxUint64 {
|
|
|
|
return math.MaxUint64
|
|
|
|
}
|
|
|
|
return uint64(x)
|
2016-12-27 19:17:21 +00:00
|
|
|
}
|
|
|
|
|
2017-12-07 21:44:19 +00:00
|
|
|
func (m *Meter) logBytes(direction, name string, read, total int64) {
|
|
|
|
m.fileIndexMutex.Lock()
|
|
|
|
idx := m.fileIndex[name]
|
2017-12-07 22:13:11 +00:00
|
|
|
logger := m.Logger
|
2017-12-07 21:44:19 +00:00
|
|
|
m.fileIndexMutex.Unlock()
|
2017-12-07 22:13:11 +00:00
|
|
|
if logger == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-12-07 21:44:19 +00:00
|
|
|
line := fmt.Sprintf("%s %d/%d %d/%d %s\n", direction, idx, m.estimatedFiles, read, total, name)
|
2017-12-07 22:13:11 +00:00
|
|
|
if err := m.Logger.Write([]byte(line)); err != nil {
|
|
|
|
m.fileIndexMutex.Lock()
|
|
|
|
m.Logger = nil
|
|
|
|
m.fileIndexMutex.Unlock()
|
2016-12-27 19:17:21 +00:00
|
|
|
}
|
|
|
|
}
|