Teach manifest how to configure MaxRetries
This commit is contained in:
parent
c1df4962ae
commit
78339e7811
@ -8,6 +8,10 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Manifest struct {
|
type Manifest struct {
|
||||||
|
// MaxRetries is the maximum number of retries a single object can
|
||||||
|
// attempt to make before it will be dropped.
|
||||||
|
MaxRetries int `git:"lfs.transfer.maxretries"`
|
||||||
|
|
||||||
basicTransfersOnly bool
|
basicTransfersOnly bool
|
||||||
downloadAdapterFuncs map[string]NewAdapterFunc
|
downloadAdapterFuncs map[string]NewAdapterFunc
|
||||||
uploadAdapterFuncs map[string]NewAdapterFunc
|
uploadAdapterFuncs map[string]NewAdapterFunc
|
||||||
@ -22,6 +26,16 @@ func NewManifest() *Manifest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func ConfigureManifest(m *Manifest, cfg *config.Configuration) *Manifest {
|
func ConfigureManifest(m *Manifest, cfg *config.Configuration) *Manifest {
|
||||||
|
if err := cfg.Unmarshal(m); err != nil {
|
||||||
|
tracerx.Printf("manifest: error parsing config, falling back to default values...: %v", err)
|
||||||
|
m.MaxRetries = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if m.MaxRetries < 1 {
|
||||||
|
tracerx.Printf("manifest: invalid retry count: %d, defaulting to %d", m.MaxRetries, 1)
|
||||||
|
m.MaxRetries = 1
|
||||||
|
}
|
||||||
|
|
||||||
m.basicTransfersOnly = cfg.BasicTransfersOnly()
|
m.basicTransfersOnly = cfg.BasicTransfersOnly()
|
||||||
|
|
||||||
configureBasicDownloadAdapter(m)
|
configureBasicDownloadAdapter(m)
|
||||||
|
@ -26,8 +26,6 @@ type Transferable interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type retryCounter struct {
|
type retryCounter struct {
|
||||||
// MaxRetries is the maximum number of retries a single object can
|
|
||||||
// attempt to make before it will be dropped.
|
|
||||||
MaxRetries int `git:"lfs.transfer.maxretries"`
|
MaxRetries int `git:"lfs.transfer.maxretries"`
|
||||||
|
|
||||||
// cmu guards count
|
// cmu guards count
|
||||||
@ -42,24 +40,11 @@ type retryCounter struct {
|
|||||||
//
|
//
|
||||||
// If it encountered an error in Unmarshaling the *config.Configuration, it will
|
// If it encountered an error in Unmarshaling the *config.Configuration, it will
|
||||||
// be returned, otherwise nil.
|
// be returned, otherwise nil.
|
||||||
func newRetryCounter(cfg *config.Configuration) *retryCounter {
|
func newRetryCounter() *retryCounter {
|
||||||
rc := &retryCounter{
|
return &retryCounter{
|
||||||
MaxRetries: defaultMaxRetries,
|
MaxRetries: defaultMaxRetries,
|
||||||
|
count: make(map[string]int),
|
||||||
count: make(map[string]int),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cfg.Unmarshal(rc); err != nil {
|
|
||||||
tracerx.Printf("rc: error parsing config, falling back to default values...: %v", err)
|
|
||||||
rc.MaxRetries = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if rc.MaxRetries < 1 {
|
|
||||||
tracerx.Printf("rc: invalid retry count: %d, defaulting to %d", rc.MaxRetries, 1)
|
|
||||||
rc.MaxRetries = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
return rc
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Increment increments the number of retries for a given OID. It is safe to
|
// Increment increments the number of retries for a given OID. It is safe to
|
||||||
@ -173,13 +158,15 @@ func NewTransferQueue(dir Direction, options ...Option) *TransferQueue {
|
|||||||
transferables: make(map[string]Transferable),
|
transferables: make(map[string]Transferable),
|
||||||
trMutex: &sync.Mutex{},
|
trMutex: &sync.Mutex{},
|
||||||
manifest: ConfigureManifest(NewManifest(), config.Config),
|
manifest: ConfigureManifest(NewManifest(), config.Config),
|
||||||
rc: newRetryCounter(config.Config),
|
rc: newRetryCounter(),
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, opt := range options {
|
for _, opt := range options {
|
||||||
opt(q)
|
opt(q)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
q.rc.MaxRetries = q.manifest.MaxRetries
|
||||||
|
|
||||||
if q.batchSize <= 0 {
|
if q.batchSize <= 0 {
|
||||||
q.batchSize = defaultBatchSize
|
q.batchSize = defaultBatchSize
|
||||||
}
|
}
|
||||||
|
@ -7,56 +7,58 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRetryCounterDefaultsToFixedRetries(t *testing.T) {
|
func TestManifestDefaultsToFixedRetries(t *testing.T) {
|
||||||
rc := newRetryCounter(config.NewFrom(config.Values{}))
|
cfg := config.NewFrom(config.Values{})
|
||||||
|
m := ConfigureManifest(NewManifest(), cfg)
|
||||||
assert.Equal(t, 1, rc.MaxRetries)
|
assert.Equal(t, 1, m.MaxRetries)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRetryCounterIsConfigurable(t *testing.T) {
|
func TestManifestIsConfigurable(t *testing.T) {
|
||||||
rc := newRetryCounter(config.NewFrom(config.Values{
|
cfg := config.NewFrom(config.Values{
|
||||||
Git: map[string]string{
|
Git: map[string]string{
|
||||||
"lfs.transfer.maxretries": "3",
|
"lfs.transfer.maxretries": "3",
|
||||||
},
|
},
|
||||||
}))
|
})
|
||||||
|
m := ConfigureManifest(NewManifest(), cfg)
|
||||||
assert.Equal(t, 3, rc.MaxRetries)
|
assert.Equal(t, 3, m.MaxRetries)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRetryCounterClampsValidValues(t *testing.T) {
|
func TestManifestClampsValidValues(t *testing.T) {
|
||||||
rc := newRetryCounter(config.NewFrom(config.Values{
|
cfg := config.NewFrom(config.Values{
|
||||||
Git: map[string]string{
|
Git: map[string]string{
|
||||||
"lfs.transfer.maxretries": "-1",
|
"lfs.transfer.maxretries": "-1",
|
||||||
},
|
},
|
||||||
}))
|
})
|
||||||
|
m := ConfigureManifest(NewManifest(), cfg)
|
||||||
assert.Equal(t, 1, rc.MaxRetries)
|
assert.Equal(t, 1, m.MaxRetries)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRetryCounterIgnoresNonInts(t *testing.T) {
|
func TestManifestIgnoresNonInts(t *testing.T) {
|
||||||
rc := newRetryCounter(config.NewFrom(config.Values{
|
cfg := config.NewFrom(config.Values{
|
||||||
Git: map[string]string{
|
Git: map[string]string{
|
||||||
"lfs.transfer.maxretries": "not_an_int",
|
"lfs.transfer.maxretries": "not_an_int",
|
||||||
},
|
},
|
||||||
}))
|
})
|
||||||
|
m := ConfigureManifest(NewManifest(), cfg)
|
||||||
|
assert.Equal(t, 1, m.MaxRetries)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetryCounterDefaultsToFixedRetries(t *testing.T) {
|
||||||
|
rc := newRetryCounter()
|
||||||
assert.Equal(t, 1, rc.MaxRetries)
|
assert.Equal(t, 1, rc.MaxRetries)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRetryCounterIncrementsObjects(t *testing.T) {
|
func TestRetryCounterIncrementsObjects(t *testing.T) {
|
||||||
rc := newRetryCounter(config.NewFrom(config.Values{}))
|
rc := newRetryCounter()
|
||||||
|
|
||||||
rc.Increment("oid")
|
rc.Increment("oid")
|
||||||
|
|
||||||
assert.Equal(t, 1, rc.CountFor("oid"))
|
assert.Equal(t, 1, rc.CountFor("oid"))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRetryCounterCanNotRetryAfterExceedingRetryCount(t *testing.T) {
|
func TestRetryCounterCanNotRetryAfterExceedingRetryCount(t *testing.T) {
|
||||||
rc := newRetryCounter(config.NewFrom(config.Values{}))
|
rc := newRetryCounter()
|
||||||
|
|
||||||
rc.Increment("oid")
|
rc.Increment("oid")
|
||||||
count, canRetry := rc.CanRetry("oid")
|
|
||||||
|
|
||||||
|
count, canRetry := rc.CanRetry("oid")
|
||||||
assert.Equal(t, 1, count)
|
assert.Equal(t, 1, count)
|
||||||
assert.False(t, canRetry)
|
assert.False(t, canRetry)
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user