lfs/tq: sort batches by descending object size
This commit is contained in:
parent
8c8dbdbb0e
commit
f17a07ea70
@ -1,6 +1,7 @@
|
||||
package lfs
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/git-lfs/git-lfs/api"
|
||||
@ -238,6 +239,11 @@ func (q *TransferQueue) collectBatches() {
|
||||
batch = append(batch, t)
|
||||
}
|
||||
|
||||
// Before enqueuing the next batch, sort it's items by size
|
||||
// (largest first) so that workers can process the largest
|
||||
// transfers first.
|
||||
sort.Sort(sort.Reverse(batch))
|
||||
|
||||
retries, err := q.enqueueAndCollectRetriesFor(batch)
|
||||
if err != nil {
|
||||
q.errorc <- err
|
||||
|
@ -64,3 +64,40 @@ begin_test "batch transfer"
|
||||
assert_pointer "master" "a.dat" "$contents_oid" 1
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "batch transfers occur in reverse order by size"
|
||||
(
|
||||
set -e
|
||||
|
||||
reponame="batch-order-test"
|
||||
setup_remote_repo "$reponame"
|
||||
clone_repo "$reponame" "$reponame"
|
||||
|
||||
git lfs track "*.dat"
|
||||
git add .gitattributes
|
||||
git commit -m "initial commit"
|
||||
|
||||
small_contents="small"
|
||||
small_oid="$(calc_oid "$small_contents")"
|
||||
printf "$small_contents" > small.dat
|
||||
|
||||
bigger_contents="bigger"
|
||||
bigger_oid="$(calc_oid "$bigger_contents")"
|
||||
printf "$bigger_contents" > bigger.dat
|
||||
|
||||
git add *.dat
|
||||
git commit -m "add small and large objects"
|
||||
|
||||
GIT_CURL_VERBOSE=1 git push origin master 2>&1 | tee push.log
|
||||
|
||||
batch="$(grep "{\"operation\":\"upload\"" push.log | head -1)"
|
||||
|
||||
pos_small="$(substring_position "$batch" "$small_oid")"
|
||||
pos_large="$(substring_position "$batch" "$bigger_oid")"
|
||||
|
||||
# Assert that the the larger object shows up earlier in the batch than the
|
||||
# smaller object
|
||||
[ "$pos_large" -lt "$pos_small" ]
|
||||
)
|
||||
end_test
|
||||
|
||||
|
@ -281,6 +281,24 @@ setup_remote_repo_with_file() {
|
||||
grep "master -> master" push.log
|
||||
}
|
||||
|
||||
# substring_position returns the position of a substring in a 1-indexed search
|
||||
# space.
|
||||
#
|
||||
# [ "$(substring_position "foo bar baz" "baz")" -eq "9" ]
|
||||
substring_position() {
|
||||
local str="$1"
|
||||
local substr="$2"
|
||||
|
||||
# 1) Print the string...
|
||||
# 2) Remove the substring and everything after it
|
||||
# 3) Count the number of characters (bytes) left, i.e., the offset of the
|
||||
# string we were looking for.
|
||||
|
||||
echo "$str" \
|
||||
| sed "s/$substr.*$//" \
|
||||
| wc -c
|
||||
}
|
||||
|
||||
# setup initializes the clean, isolated environment for integration tests.
|
||||
setup() {
|
||||
cd "$ROOTDIR"
|
||||
|
Loading…
Reference in New Issue
Block a user