95 lines
2.4 KiB
Go
95 lines
2.4 KiB
Go
package lfs
|
|
|
|
import "sync/atomic"
|
|
|
|
// Batcher provides a way to process a set of items in groups of n. Items can
|
|
// be added to the batcher from multiple goroutines and pulled off in groups
|
|
// when one of the following conditions occurs:
|
|
// * The batch size is reached
|
|
// * Exit() is called
|
|
// When an Exit() occurs, the group may be smaller than the batch size.
|
|
type Batcher struct {
|
|
exited uint32
|
|
batchSize int
|
|
input chan interface{}
|
|
batchReady chan []interface{}
|
|
truncate chan interface{}
|
|
}
|
|
|
|
// NewBatcher creates a Batcher with the batchSize.
|
|
func NewBatcher(batchSize int) *Batcher {
|
|
b := &Batcher{
|
|
batchSize: batchSize,
|
|
input: make(chan interface{}),
|
|
batchReady: make(chan []interface{}),
|
|
truncate: make(chan interface{}),
|
|
}
|
|
|
|
go b.acceptInput()
|
|
return b
|
|
}
|
|
|
|
// Add adds an item to the batcher. Add is safe to call from multiple
|
|
// goroutines.
|
|
func (b *Batcher) Add(t interface{}) {
|
|
if atomic.CompareAndSwapUint32(&b.exited, 1, 0) {
|
|
b.input = make(chan interface{})
|
|
b.truncate = make(chan interface{})
|
|
go b.acceptInput()
|
|
}
|
|
|
|
b.input <- t
|
|
}
|
|
|
|
// Next will wait for the one of the above batch triggers to occur and return
|
|
// the accumulated batch.
|
|
func (b *Batcher) Next() []interface{} {
|
|
return <-b.batchReady
|
|
}
|
|
|
|
// Truncate causes the current batch to halt accumulation and return
|
|
// immediately, even if it is smaller than the given batch size.
|
|
func (b *Batcher) Truncate() {
|
|
b.truncate <- struct{}{}
|
|
}
|
|
|
|
// Exit stops all batching and allows Next() to return. Calling Add() after
|
|
// calling Exit() will reset the batcher.
|
|
func (b *Batcher) Exit() {
|
|
atomic.StoreUint32(&b.exited, 1)
|
|
close(b.input)
|
|
close(b.truncate)
|
|
}
|
|
|
|
// acceptInput runs in its own goroutine and accepts input from external
|
|
// clients. Without truncation, it fills and dispenses batches in a sequential
|
|
// order: for a batch size N, N items will be processed before a new batch is
|
|
// ready. If a batch is truncated while still filling itself, it will be
|
|
// returned immediately, opening up a new batch for all subsequent items.
|
|
func (b *Batcher) acceptInput() {
|
|
var exit bool
|
|
|
|
for {
|
|
batch := make([]interface{}, 0, b.batchSize)
|
|
Acc:
|
|
for len(batch) < b.batchSize {
|
|
select {
|
|
case t, ok := <-b.input:
|
|
if !ok {
|
|
exit = true // input channel was closed by Exit()
|
|
break Acc
|
|
}
|
|
batch = append(batch, t)
|
|
case <-b.truncate:
|
|
break Acc
|
|
}
|
|
}
|
|
|
|
b.batchReady <- batch
|
|
|
|
if exit {
|
|
return
|
|
}
|
|
}
|
|
}
|