lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tq,test: export func for calling Batch API directly

Browse Source
This commit is contained in:
risk danger olson 2017-01-06 20:13:31 -07:00
parent 90878099b2
commit 924befe2f0
5 changed files with 63 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
test/git-lfs-test-server-api/main.go
View File

@ -10,7 +10,6 @@ import (
"strconv"
"strings"
"github.com/git-lfs/git-lfs/api"
"github.com/git-lfs/git-lfs/config"
"github.com/git-lfs/git-lfs/errors"
"github.com/git-lfs/git-lfs/lfs"
@ -28,7 +27,7 @@ type TestObject struct {
type ServerTest struct {
Name string
F func(oidsExist, oidsMissing []TestObject) error
F func(m *tq.Manifest, oidsExist, oidsMissing []TestObject) error
}
var (
@ -65,6 +64,11 @@ func testServerApi(cmd *cobra.Command, args []string) {
// Force loading of config before we alter it
config.Config.Git.All()
manifest, err := buildManifest()
if err != nil {
exit("error building tq.Manifest: " + err.Error())
}
// Configure the endpoint manually
var endp lfsapi.Endpoint
finder := lfsapi.NewEndpointFinder(config.Config.Git)
@ -83,7 +87,7 @@ func testServerApi(cmd *cobra.Command, args []string) {
} else {
fmt.Printf("Creating test data (will upload to server)\n")
var err error
oidsExist, oidsMissing, err = buildTestData()
oidsExist, oidsMissing, err = buildTestData(manifest)
if err != nil {
exit("Failed to set up test data, aborting")
}
@ -97,7 +101,7 @@ func testServerApi(cmd *cobra.Command, args []string) {
}
ok := runTests(oidsExist, oidsMissing)
ok := runTests(manifest, oidsExist, oidsMissing)
if !ok {
exit("One or more tests failed, see above")
}
@ -136,13 +140,16 @@ func (*testDataCallback) Errorf(format string, args ...interface{}) {
fmt.Printf(format, args...)
}
func buildTestData() (oidsExist, oidsMissing []TestObject, err error) {
func buildManifest() (*tq.Manifest, error) {
cfg := config.Config
apiClient, err := lfsapi.NewClient(cfg.Os, cfg.Git)
if err != nil {
return nil, nil, err
return nil, err
}
return tq.NewManifestWithClient(apiClient), nil
}
func buildTestData(manifest *tq.Manifest) (oidsExist, oidsMissing []TestObject, err error) {
const oidCount = 50
oidsExist = make([]TestObject, 0, oidCount)
oidsMissing = make([]TestObject, 0, oidCount)
@ -166,8 +173,7 @@ func buildTestData() (oidsExist, oidsMissing []TestObject, err error) {
outputs := repo.AddCommits([]*test.CommitInput{&commit})
// now upload
manifest := tq.NewManifestWithClient(apiClient)
uploadQueue := tq.NewTransferQueue(tq.Upload, manifest, "", tq.WithProgress(meter))
uploadQueue := tq.NewTransferQueue(tq.Upload, manifest, "origin", tq.WithProgress(meter))
for _, f := range outputs[0].Files {
oidsExist = append(oidsExist, TestObject{Oid: f.Oid, Size: f.Size})
@ -211,12 +217,11 @@ func saveTestOids(filename string, objs []TestObject) {
}
func runTests(oidsExist, oidsMissing []TestObject) bool {
func runTests(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) bool {
ok := true
fmt.Printf("Running %d tests...\n", len(tests))
for _, t := range tests {
err := runTest(t, oidsExist, oidsMissing)
err := runTest(t, manifest, oidsExist, oidsMissing)
if err != nil {
ok = false
}
@ -224,7 +229,7 @@ func runTests(oidsExist, oidsMissing []TestObject) bool {
return ok
}
func runTest(t ServerTest, oidsExist, oidsMissing []TestObject) error {
func runTest(t ServerTest, manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error {
const linelen = 70
line := t.Name
if len(line) > linelen {
@ -234,7 +239,7 @@ func runTest(t ServerTest, oidsExist, oidsMissing []TestObject) error {
}
fmt.Printf("%s...\r", line)
err := t.F(oidsExist, oidsMissing)
err := t.F(manifest, oidsExist, oidsMissing)
if err != nil {
fmt.Printf("%s FAILED\n", line)
fmt.Println(err.Error())
@ -250,21 +255,17 @@ func exit(format string, args ...interface{}) {
os.Exit(2)
}
func addTest(name string, f func(oidsExist, oidsMissing []TestObject) error) {
func addTest(name string, f func(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error) {
tests = append(tests, ServerTest{Name: name, F: f})
}
func callBatchApi(op string, objs []TestObject) ([]*api.ObjectResource, error) {
apiobjs := make([]*api.ObjectResource, 0, len(objs))
func callBatchApi(manifest *tq.Manifest, dir tq.Direction, objs []TestObject) ([]*tq.Transfer, error) {
apiobjs := make([]*tq.Transfer, 0, len(objs))
for _, o := range objs {
apiobjs = append(apiobjs, &api.ObjectResource{Oid: o.Oid, Size: o.Size})
apiobjs = append(apiobjs, &tq.Transfer{Oid: o.Oid, Size: o.Size})
}
o, _, err := api.Batch(config.Config, apiobjs, op, []string{"basic"})
if err != nil {
return nil, err
}
return o, nil
return tq.Batch(dir, manifest, "origin", apiobjs)
}
// Combine 2 slices into one by "randomly" interleaving

16
test/git-lfs-test-server-api/testdownload.go
View File

@ -6,11 +6,12 @@ import (
"fmt"
"github.com/git-lfs/git-lfs/tools"
"github.com/git-lfs/git-lfs/tq"
)
// "download" - all present
func downloadAllExist(oidsExist, oidsMissing []TestObject) error {
retobjs, err := callBatchApi("download", oidsExist)
func downloadAllExist(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error {
retobjs, err := callBatchApi(manifest, tq.Download, oidsExist)
if err != nil {
return err
@ -36,8 +37,8 @@ func downloadAllExist(oidsExist, oidsMissing []TestObject) error {
}
// "download" - all missing (test includes 404 error entry)
func downloadAllMissing(oidsExist, oidsMissing []TestObject) error {
retobjs, err := callBatchApi("download", oidsMissing)
func downloadAllMissing(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error {
retobjs, err := callBatchApi(manifest, tq.Download, oidsMissing)
if err != nil {
return err
@ -54,7 +55,7 @@ func downloadAllMissing(oidsExist, oidsMissing []TestObject) error {
errbuf.WriteString(fmt.Sprintf("Download link should not exist for %s, was %s\n", o.Oid, link))
}
if o.Error == nil {
errbuf.WriteString(fmt.Sprintf("Download should include an error for missing object %s, was %s\n", o.Oid))
errbuf.WriteString(fmt.Sprintf("Download should include an error for missing object %s\n", o.Oid))
} else if o.Error.Code != 404 {
errbuf.WriteString(fmt.Sprintf("Download error code for missing object %s should be 404, got %d\n", o.Oid, o.Error.Code))
}
@ -68,8 +69,7 @@ func downloadAllMissing(oidsExist, oidsMissing []TestObject) error {
}
// "download" - mixture
func downloadMixed(oidsExist, oidsMissing []TestObject) error {
func downloadMixed(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error {
existSet := tools.NewStringSetWithCapacity(len(oidsExist))
for _, o := range oidsExist {
existSet.Add(o.Oid)
@ -80,7 +80,7 @@ func downloadMixed(oidsExist, oidsMissing []TestObject) error {
}
calloids := interleaveTestData(oidsExist, oidsMissing)
retobjs, err := callBatchApi("download", calloids)
retobjs, err := callBatchApi(manifest, tq.Download, calloids)
if err != nil {
return err

18
test/git-lfs-test-server-api/testupload.go
View File

@ -6,11 +6,12 @@ import (
"fmt"
"github.com/git-lfs/git-lfs/tools"
"github.com/git-lfs/git-lfs/tq"
)
// "upload" - all missing
func uploadAllMissing(oidsExist, oidsMissing []TestObject) error {
retobjs, err := callBatchApi("upload", oidsMissing)
func uploadAllMissing(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error {
retobjs, err := callBatchApi(manifest, tq.Upload, oidsMissing)
if err != nil {
return err
@ -37,8 +38,8 @@ func uploadAllMissing(oidsExist, oidsMissing []TestObject) error {
}
// "upload" - all present
func uploadAllExists(oidsExist, oidsMissing []TestObject) error {
retobjs, err := callBatchApi("upload", oidsExist)
func uploadAllExists(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error {
retobjs, err := callBatchApi(manifest, tq.Upload, oidsExist)
if err != nil {
return err
@ -64,8 +65,7 @@ func uploadAllExists(oidsExist, oidsMissing []TestObject) error {
}
// "upload" - mix of missing & present
func uploadMixed(oidsExist, oidsMissing []TestObject) error {
func uploadMixed(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error {
existSet := tools.NewStringSetWithCapacity(len(oidsExist))
for _, o := range oidsExist {
existSet.Add(o.Oid)
@ -76,7 +76,7 @@ func uploadMixed(oidsExist, oidsMissing []TestObject) error {
}
calloids := interleaveTestData(oidsExist, oidsMissing)
retobjs, err := callBatchApi("upload", calloids)
retobjs, err := callBatchApi(manifest, tq.Upload, calloids)
if err != nil {
return err
@ -109,7 +109,7 @@ func uploadMixed(oidsExist, oidsMissing []TestObject) error {
}
func uploadEdgeCases(oidsExist, oidsMissing []TestObject) error {
func uploadEdgeCases(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error {
errorCases := make([]TestObject, 0, 5)
errorCodeMap := make(map[string]int, 5)
errorReasonMap := make(map[string]string, 5)
@ -149,7 +149,7 @@ func uploadEdgeCases(oidsExist, oidsMissing []TestObject) error {
validReasonMap[sha] = "Zero size should be allowed"
calloids := interleaveTestData(errorCases, validCases)
retobjs, err := callBatchApi("upload", calloids)
retobjs, err := callBatchApi(manifest, tq.Upload, calloids)
if err != nil {
return err

19
tq/api.go
View File

@ -25,6 +25,25 @@ type batchResponse struct {
Objects []*Transfer `json:"objects"`
}
func Batch(dir Direction, m *Manifest, remote string, objects []*Transfer) ([]*Transfer, error) {
if len(objects) == 0 {
return nil, nil
}
breq := &batchRequest{
Operation: string(dir),
Objects: objects,
TransferAdapterNames: m.GetAdapterNames(dir),
}
cli := &tqClient{Client: m.APIClient()}
bres, _, err := cli.Batch(remote, breq)
if err != nil {
return nil, err
}
return bres.Objects, nil
}
func (c *tqClient) Batch(remote string, bReq *batchRequest) (*batchResponse, *http.Response, error) {
bRes := &batchResponse{}
if len(bReq.Objects) == 0 {

6
tq/transfer.go
View File

@ -10,11 +10,11 @@ import (
"github.com/git-lfs/git-lfs/lfsapi"
)
type Direction int
type Direction string
const (
Upload = Direction(iota)
Download = Direction(iota)
Upload = Direction("upload")
Download = Direction("download")
)
type Transfer struct {