From 924befe2f088890be5102c740b7c84a09d9d6041 Mon Sep 17 00:00:00 2001 From: risk danger olson Date: Fri, 6 Jan 2017 20:13:31 -0700 Subject: [PATCH] tq,test: export func for calling Batch API directly --- test/git-lfs-test-server-api/main.go | 47 ++++++++++---------- test/git-lfs-test-server-api/testdownload.go | 16 +++---- test/git-lfs-test-server-api/testupload.go | 18 ++++---- tq/api.go | 19 ++++++++ tq/transfer.go | 6 +-- 5 files changed, 63 insertions(+), 43 deletions(-) diff --git a/test/git-lfs-test-server-api/main.go b/test/git-lfs-test-server-api/main.go index b232d73b..4f600b02 100644 --- a/test/git-lfs-test-server-api/main.go +++ b/test/git-lfs-test-server-api/main.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" - "github.com/git-lfs/git-lfs/api" "github.com/git-lfs/git-lfs/config" "github.com/git-lfs/git-lfs/errors" "github.com/git-lfs/git-lfs/lfs" @@ -28,7 +27,7 @@ type TestObject struct { type ServerTest struct { Name string - F func(oidsExist, oidsMissing []TestObject) error + F func(m *tq.Manifest, oidsExist, oidsMissing []TestObject) error } var ( @@ -65,6 +64,11 @@ func testServerApi(cmd *cobra.Command, args []string) { // Force loading of config before we alter it config.Config.Git.All() + manifest, err := buildManifest() + if err != nil { + exit("error building tq.Manifest: " + err.Error()) + } + // Configure the endpoint manually var endp lfsapi.Endpoint finder := lfsapi.NewEndpointFinder(config.Config.Git) @@ -83,7 +87,7 @@ func testServerApi(cmd *cobra.Command, args []string) { } else { fmt.Printf("Creating test data (will upload to server)\n") var err error - oidsExist, oidsMissing, err = buildTestData() + oidsExist, oidsMissing, err = buildTestData(manifest) if err != nil { exit("Failed to set up test data, aborting") } @@ -97,7 +101,7 @@ func testServerApi(cmd *cobra.Command, args []string) { } - ok := runTests(oidsExist, oidsMissing) + ok := runTests(manifest, oidsExist, oidsMissing) if !ok { exit("One or more tests failed, see above") } @@ -136,13 +140,16 @@ func (*testDataCallback) Errorf(format string, args ...interface{}) { fmt.Printf(format, args...) } -func buildTestData() (oidsExist, oidsMissing []TestObject, err error) { +func buildManifest() (*tq.Manifest, error) { cfg := config.Config apiClient, err := lfsapi.NewClient(cfg.Os, cfg.Git) if err != nil { - return nil, nil, err + return nil, err } + return tq.NewManifestWithClient(apiClient), nil +} +func buildTestData(manifest *tq.Manifest) (oidsExist, oidsMissing []TestObject, err error) { const oidCount = 50 oidsExist = make([]TestObject, 0, oidCount) oidsMissing = make([]TestObject, 0, oidCount) @@ -166,8 +173,7 @@ func buildTestData() (oidsExist, oidsMissing []TestObject, err error) { outputs := repo.AddCommits([]*test.CommitInput{&commit}) // now upload - manifest := tq.NewManifestWithClient(apiClient) - uploadQueue := tq.NewTransferQueue(tq.Upload, manifest, "", tq.WithProgress(meter)) + uploadQueue := tq.NewTransferQueue(tq.Upload, manifest, "origin", tq.WithProgress(meter)) for _, f := range outputs[0].Files { oidsExist = append(oidsExist, TestObject{Oid: f.Oid, Size: f.Size}) @@ -211,12 +217,11 @@ func saveTestOids(filename string, objs []TestObject) { } -func runTests(oidsExist, oidsMissing []TestObject) bool { - +func runTests(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) bool { ok := true fmt.Printf("Running %d tests...\n", len(tests)) for _, t := range tests { - err := runTest(t, oidsExist, oidsMissing) + err := runTest(t, manifest, oidsExist, oidsMissing) if err != nil { ok = false } @@ -224,7 +229,7 @@ func runTests(oidsExist, oidsMissing []TestObject) bool { return ok } -func runTest(t ServerTest, oidsExist, oidsMissing []TestObject) error { +func runTest(t ServerTest, manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error { const linelen = 70 line := t.Name if len(line) > linelen { @@ -234,7 +239,7 @@ func runTest(t ServerTest, oidsExist, oidsMissing []TestObject) error { } fmt.Printf("%s...\r", line) - err := t.F(oidsExist, oidsMissing) + err := t.F(manifest, oidsExist, oidsMissing) if err != nil { fmt.Printf("%s FAILED\n", line) fmt.Println(err.Error()) @@ -250,21 +255,17 @@ func exit(format string, args ...interface{}) { os.Exit(2) } -func addTest(name string, f func(oidsExist, oidsMissing []TestObject) error) { +func addTest(name string, f func(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error) { tests = append(tests, ServerTest{Name: name, F: f}) } -func callBatchApi(op string, objs []TestObject) ([]*api.ObjectResource, error) { - - apiobjs := make([]*api.ObjectResource, 0, len(objs)) +func callBatchApi(manifest *tq.Manifest, dir tq.Direction, objs []TestObject) ([]*tq.Transfer, error) { + apiobjs := make([]*tq.Transfer, 0, len(objs)) for _, o := range objs { - apiobjs = append(apiobjs, &api.ObjectResource{Oid: o.Oid, Size: o.Size}) + apiobjs = append(apiobjs, &tq.Transfer{Oid: o.Oid, Size: o.Size}) } - o, _, err := api.Batch(config.Config, apiobjs, op, []string{"basic"}) - if err != nil { - return nil, err - } - return o, nil + + return tq.Batch(dir, manifest, "origin", apiobjs) } // Combine 2 slices into one by "randomly" interleaving diff --git a/test/git-lfs-test-server-api/testdownload.go b/test/git-lfs-test-server-api/testdownload.go index 25e03e25..74d579a4 100644 --- a/test/git-lfs-test-server-api/testdownload.go +++ b/test/git-lfs-test-server-api/testdownload.go @@ -6,11 +6,12 @@ import ( "fmt" "github.com/git-lfs/git-lfs/tools" + "github.com/git-lfs/git-lfs/tq" ) // "download" - all present -func downloadAllExist(oidsExist, oidsMissing []TestObject) error { - retobjs, err := callBatchApi("download", oidsExist) +func downloadAllExist(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error { + retobjs, err := callBatchApi(manifest, tq.Download, oidsExist) if err != nil { return err @@ -36,8 +37,8 @@ func downloadAllExist(oidsExist, oidsMissing []TestObject) error { } // "download" - all missing (test includes 404 error entry) -func downloadAllMissing(oidsExist, oidsMissing []TestObject) error { - retobjs, err := callBatchApi("download", oidsMissing) +func downloadAllMissing(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error { + retobjs, err := callBatchApi(manifest, tq.Download, oidsMissing) if err != nil { return err @@ -54,7 +55,7 @@ func downloadAllMissing(oidsExist, oidsMissing []TestObject) error { errbuf.WriteString(fmt.Sprintf("Download link should not exist for %s, was %s\n", o.Oid, link)) } if o.Error == nil { - errbuf.WriteString(fmt.Sprintf("Download should include an error for missing object %s, was %s\n", o.Oid)) + errbuf.WriteString(fmt.Sprintf("Download should include an error for missing object %s\n", o.Oid)) } else if o.Error.Code != 404 { errbuf.WriteString(fmt.Sprintf("Download error code for missing object %s should be 404, got %d\n", o.Oid, o.Error.Code)) } @@ -68,8 +69,7 @@ func downloadAllMissing(oidsExist, oidsMissing []TestObject) error { } // "download" - mixture -func downloadMixed(oidsExist, oidsMissing []TestObject) error { - +func downloadMixed(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error { existSet := tools.NewStringSetWithCapacity(len(oidsExist)) for _, o := range oidsExist { existSet.Add(o.Oid) @@ -80,7 +80,7 @@ func downloadMixed(oidsExist, oidsMissing []TestObject) error { } calloids := interleaveTestData(oidsExist, oidsMissing) - retobjs, err := callBatchApi("download", calloids) + retobjs, err := callBatchApi(manifest, tq.Download, calloids) if err != nil { return err diff --git a/test/git-lfs-test-server-api/testupload.go b/test/git-lfs-test-server-api/testupload.go index f72a40f1..ecefb10a 100644 --- a/test/git-lfs-test-server-api/testupload.go +++ b/test/git-lfs-test-server-api/testupload.go @@ -6,11 +6,12 @@ import ( "fmt" "github.com/git-lfs/git-lfs/tools" + "github.com/git-lfs/git-lfs/tq" ) // "upload" - all missing -func uploadAllMissing(oidsExist, oidsMissing []TestObject) error { - retobjs, err := callBatchApi("upload", oidsMissing) +func uploadAllMissing(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error { + retobjs, err := callBatchApi(manifest, tq.Upload, oidsMissing) if err != nil { return err @@ -37,8 +38,8 @@ func uploadAllMissing(oidsExist, oidsMissing []TestObject) error { } // "upload" - all present -func uploadAllExists(oidsExist, oidsMissing []TestObject) error { - retobjs, err := callBatchApi("upload", oidsExist) +func uploadAllExists(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error { + retobjs, err := callBatchApi(manifest, tq.Upload, oidsExist) if err != nil { return err @@ -64,8 +65,7 @@ func uploadAllExists(oidsExist, oidsMissing []TestObject) error { } // "upload" - mix of missing & present -func uploadMixed(oidsExist, oidsMissing []TestObject) error { - +func uploadMixed(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error { existSet := tools.NewStringSetWithCapacity(len(oidsExist)) for _, o := range oidsExist { existSet.Add(o.Oid) @@ -76,7 +76,7 @@ func uploadMixed(oidsExist, oidsMissing []TestObject) error { } calloids := interleaveTestData(oidsExist, oidsMissing) - retobjs, err := callBatchApi("upload", calloids) + retobjs, err := callBatchApi(manifest, tq.Upload, calloids) if err != nil { return err @@ -109,7 +109,7 @@ func uploadMixed(oidsExist, oidsMissing []TestObject) error { } -func uploadEdgeCases(oidsExist, oidsMissing []TestObject) error { +func uploadEdgeCases(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error { errorCases := make([]TestObject, 0, 5) errorCodeMap := make(map[string]int, 5) errorReasonMap := make(map[string]string, 5) @@ -149,7 +149,7 @@ func uploadEdgeCases(oidsExist, oidsMissing []TestObject) error { validReasonMap[sha] = "Zero size should be allowed" calloids := interleaveTestData(errorCases, validCases) - retobjs, err := callBatchApi("upload", calloids) + retobjs, err := callBatchApi(manifest, tq.Upload, calloids) if err != nil { return err diff --git a/tq/api.go b/tq/api.go index 72d19277..a4ee240f 100644 --- a/tq/api.go +++ b/tq/api.go @@ -25,6 +25,25 @@ type batchResponse struct { Objects []*Transfer `json:"objects"` } +func Batch(dir Direction, m *Manifest, remote string, objects []*Transfer) ([]*Transfer, error) { + if len(objects) == 0 { + return nil, nil + } + + breq := &batchRequest{ + Operation: string(dir), + Objects: objects, + TransferAdapterNames: m.GetAdapterNames(dir), + } + + cli := &tqClient{Client: m.APIClient()} + bres, _, err := cli.Batch(remote, breq) + if err != nil { + return nil, err + } + return bres.Objects, nil +} + func (c *tqClient) Batch(remote string, bReq *batchRequest) (*batchResponse, *http.Response, error) { bRes := &batchResponse{} if len(bReq.Objects) == 0 { diff --git a/tq/transfer.go b/tq/transfer.go index aae0c968..6cf3080b 100644 --- a/tq/transfer.go +++ b/tq/transfer.go @@ -10,11 +10,11 @@ import ( "github.com/git-lfs/git-lfs/lfsapi" ) -type Direction int +type Direction string const ( - Upload = Direction(iota) - Download = Direction(iota) + Upload = Direction("upload") + Download = Direction("download") ) type Transfer struct {