2015-11-24 11:55:33 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bufio"
|
|
|
|
"crypto/sha256"
|
|
|
|
"encoding/hex"
|
|
|
|
"fmt"
|
|
|
|
"math/rand"
|
|
|
|
"os"
|
2015-11-24 14:26:00 +00:00
|
|
|
"strconv"
|
2015-11-24 11:55:33 +00:00
|
|
|
"strings"
|
|
|
|
|
2016-11-15 17:01:18 +00:00
|
|
|
"github.com/git-lfs/git-lfs/errors"
|
2017-10-25 18:20:25 +00:00
|
|
|
"github.com/git-lfs/git-lfs/fs"
|
2016-12-19 18:45:22 +00:00
|
|
|
"github.com/git-lfs/git-lfs/lfsapi"
|
2016-12-07 02:59:42 +00:00
|
|
|
"github.com/git-lfs/git-lfs/progress"
|
2016-11-15 17:01:18 +00:00
|
|
|
"github.com/git-lfs/git-lfs/test"
|
2016-12-12 00:20:14 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tq"
|
2016-05-23 18:02:27 +00:00
|
|
|
"github.com/spf13/cobra"
|
2015-11-24 11:55:33 +00:00
|
|
|
)
|
|
|
|
|
2015-11-24 14:26:00 +00:00
|
|
|
type TestObject struct {
|
|
|
|
Oid string
|
|
|
|
Size int64
|
|
|
|
}
|
|
|
|
|
2015-11-24 11:55:33 +00:00
|
|
|
type ServerTest struct {
|
|
|
|
Name string
|
2017-01-07 03:13:31 +00:00
|
|
|
F func(m *tq.Manifest, oidsExist, oidsMissing []TestObject) error
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
|
|
|
RootCmd = &cobra.Command{
|
|
|
|
Use: "git-lfs-test-server-api [--url=<apiurl> | --clone=<cloneurl>] [<oid-exists-file> <oid-missing-file>]",
|
|
|
|
Short: "Test a Git LFS API server for compliance",
|
|
|
|
Run: testServerApi,
|
|
|
|
}
|
2015-11-25 14:41:31 +00:00
|
|
|
apiUrl string
|
|
|
|
cloneUrl string
|
|
|
|
savePrefix string
|
2015-11-24 11:55:33 +00:00
|
|
|
|
|
|
|
tests []ServerTest
|
|
|
|
)
|
|
|
|
|
|
|
|
func main() {
|
|
|
|
RootCmd.Execute()
|
|
|
|
}
|
|
|
|
|
|
|
|
func testServerApi(cmd *cobra.Command, args []string) {
|
|
|
|
if (len(apiUrl) == 0 && len(cloneUrl) == 0) ||
|
|
|
|
(len(apiUrl) != 0 && len(cloneUrl) != 0) {
|
|
|
|
exit("Must supply either --url or --clone (and not both)")
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(args) != 0 && len(args) != 2 {
|
|
|
|
exit("Must supply either no file arguments or both the exists AND missing file")
|
|
|
|
}
|
|
|
|
|
2015-11-25 14:41:31 +00:00
|
|
|
if len(args) != 0 && len(savePrefix) > 0 {
|
|
|
|
exit("Cannot combine input files and --save option")
|
|
|
|
}
|
|
|
|
|
2017-10-25 18:20:25 +00:00
|
|
|
// Build test data for existing files & upload
|
|
|
|
// Use test repo for this to simplify the process of making sure data matches oid
|
|
|
|
// We're not performing a real test at this point (although an upload fail will break it)
|
|
|
|
var callback testDataCallback
|
|
|
|
repo := test.NewRepo(&callback)
|
|
|
|
|
2015-11-25 10:29:54 +00:00
|
|
|
// Force loading of config before we alter it
|
2017-10-25 18:20:25 +00:00
|
|
|
repo.GitEnv().All()
|
|
|
|
repo.Pushd()
|
|
|
|
defer repo.Popd()
|
2015-11-25 10:29:54 +00:00
|
|
|
|
2017-10-25 18:20:25 +00:00
|
|
|
manifest, err := buildManifest(repo)
|
2017-01-07 03:13:31 +00:00
|
|
|
if err != nil {
|
|
|
|
exit("error building tq.Manifest: " + err.Error())
|
|
|
|
}
|
|
|
|
|
2015-11-24 14:26:00 +00:00
|
|
|
var oidsExist, oidsMissing []TestObject
|
2015-11-24 11:55:33 +00:00
|
|
|
if len(args) >= 2 {
|
|
|
|
fmt.Printf("Reading test data from files (no server content changes)\n")
|
|
|
|
oidsExist = readTestOids(args[0])
|
|
|
|
oidsMissing = readTestOids(args[1])
|
|
|
|
} else {
|
2015-11-25 10:35:01 +00:00
|
|
|
fmt.Printf("Creating test data (will upload to server)\n")
|
2015-11-24 16:37:09 +00:00
|
|
|
var err error
|
2017-10-25 18:20:25 +00:00
|
|
|
oidsExist, oidsMissing, err = buildTestData(repo, manifest)
|
2015-11-24 11:55:33 +00:00
|
|
|
if err != nil {
|
|
|
|
exit("Failed to set up test data, aborting")
|
|
|
|
}
|
2015-11-25 14:41:31 +00:00
|
|
|
if len(savePrefix) > 0 {
|
|
|
|
existFile := savePrefix + "_exists"
|
|
|
|
missingFile := savePrefix + "_missing"
|
|
|
|
saveTestOids(existFile, oidsExist)
|
|
|
|
saveTestOids(missingFile, oidsMissing)
|
|
|
|
fmt.Printf("Wrote test to %s, %s for future use\n", existFile, missingFile)
|
|
|
|
}
|
|
|
|
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
|
|
|
|
2017-01-07 03:13:31 +00:00
|
|
|
ok := runTests(manifest, oidsExist, oidsMissing)
|
2015-11-26 11:53:52 +00:00
|
|
|
if !ok {
|
|
|
|
exit("One or more tests failed, see above")
|
|
|
|
}
|
|
|
|
fmt.Println("All tests passed")
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
|
|
|
|
2015-11-24 14:26:00 +00:00
|
|
|
func readTestOids(filename string) []TestObject {
|
2015-11-24 11:55:33 +00:00
|
|
|
f, err := os.OpenFile(filename, os.O_RDONLY, 0644)
|
|
|
|
if err != nil {
|
|
|
|
exit("Error opening file %s", filename)
|
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
2015-11-24 14:26:00 +00:00
|
|
|
var ret []TestObject
|
2015-11-24 11:55:33 +00:00
|
|
|
rdr := bufio.NewReader(f)
|
|
|
|
line, err := rdr.ReadString('\n')
|
|
|
|
for err == nil {
|
2015-11-24 14:26:00 +00:00
|
|
|
fields := strings.Fields(strings.TrimSpace(line))
|
|
|
|
if len(fields) == 2 {
|
|
|
|
sz, _ := strconv.ParseInt(fields[1], 10, 64)
|
|
|
|
ret = append(ret, TestObject{Oid: fields[0], Size: sz})
|
|
|
|
}
|
|
|
|
|
2015-11-24 11:55:33 +00:00
|
|
|
line, err = rdr.ReadString('\n')
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret
|
|
|
|
}
|
|
|
|
|
2015-11-24 16:37:09 +00:00
|
|
|
type testDataCallback struct{}
|
|
|
|
|
|
|
|
func (*testDataCallback) Fatalf(format string, args ...interface{}) {
|
|
|
|
exit(format, args...)
|
|
|
|
}
|
|
|
|
func (*testDataCallback) Errorf(format string, args ...interface{}) {
|
|
|
|
fmt.Printf(format, args...)
|
|
|
|
}
|
|
|
|
|
2017-10-25 18:20:25 +00:00
|
|
|
func buildManifest(r *test.Repo) (*tq.Manifest, error) {
|
2017-04-14 16:20:15 +00:00
|
|
|
// Configure the endpoint manually
|
2017-10-25 18:20:25 +00:00
|
|
|
finder := lfsapi.NewEndpointFinder(r.GitEnv())
|
2017-04-14 16:20:15 +00:00
|
|
|
|
|
|
|
var endp lfsapi.Endpoint
|
|
|
|
if len(cloneUrl) > 0 {
|
|
|
|
endp = finder.NewEndpointFromCloneURL(cloneUrl)
|
|
|
|
} else {
|
|
|
|
endp = finder.NewEndpoint(apiUrl)
|
|
|
|
}
|
|
|
|
|
2017-10-25 18:20:25 +00:00
|
|
|
apiClient, err := lfsapi.NewClient(r.OSEnv(), r.GitEnv())
|
2017-04-14 16:20:15 +00:00
|
|
|
apiClient.Endpoints = &constantEndpoint{
|
|
|
|
e: endp,
|
|
|
|
EndpointFinder: apiClient.Endpoints,
|
|
|
|
}
|
2017-01-03 22:48:30 +00:00
|
|
|
if err != nil {
|
2017-01-07 03:13:31 +00:00
|
|
|
return nil, err
|
2017-01-03 22:48:30 +00:00
|
|
|
}
|
2017-10-25 17:46:37 +00:00
|
|
|
return tq.NewManifest(r.Filesystem(), apiClient, "", ""), nil
|
2017-01-07 03:13:31 +00:00
|
|
|
}
|
2017-01-03 22:48:30 +00:00
|
|
|
|
2017-04-14 16:19:53 +00:00
|
|
|
type constantEndpoint struct {
|
|
|
|
e lfsapi.Endpoint
|
|
|
|
|
|
|
|
lfsapi.EndpointFinder
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *constantEndpoint) NewEndpointFromCloneURL(rawurl string) lfsapi.Endpoint { return c.e }
|
|
|
|
|
|
|
|
func (c *constantEndpoint) NewEndpoint(rawurl string) lfsapi.Endpoint { return c.e }
|
|
|
|
|
|
|
|
func (c *constantEndpoint) Endpoint(operation, remote string) lfsapi.Endpoint { return c.e }
|
|
|
|
|
|
|
|
func (c *constantEndpoint) RemoteEndpoint(operation, remote string) lfsapi.Endpoint { return c.e }
|
|
|
|
|
2017-10-25 18:20:25 +00:00
|
|
|
func buildTestData(repo *test.Repo, manifest *tq.Manifest) (oidsExist, oidsMissing []TestObject, err error) {
|
2015-11-24 11:55:33 +00:00
|
|
|
const oidCount = 50
|
2015-11-24 14:26:00 +00:00
|
|
|
oidsExist = make([]TestObject, 0, oidCount)
|
|
|
|
oidsMissing = make([]TestObject, 0, oidCount)
|
2015-11-24 11:55:33 +00:00
|
|
|
|
2015-11-24 16:37:09 +00:00
|
|
|
// just one commit
|
2017-10-25 18:20:25 +00:00
|
|
|
meter := progress.NewMeter(progress.WithOSEnv(repo.OSEnv()))
|
2015-11-24 16:37:09 +00:00
|
|
|
commit := test.CommitInput{CommitterName: "A N Other", CommitterEmail: "noone@somewhere.com"}
|
|
|
|
for i := 0; i < oidCount; i++ {
|
|
|
|
filename := fmt.Sprintf("file%d.dat", i)
|
|
|
|
sz := int64(rand.Intn(200)) + 50
|
|
|
|
commit.Files = append(commit.Files, &test.FileInput{Filename: filename, Size: sz})
|
2016-12-07 23:42:50 +00:00
|
|
|
meter.Add(sz)
|
2015-11-24 16:37:09 +00:00
|
|
|
}
|
|
|
|
outputs := repo.AddCommits([]*test.CommitInput{&commit})
|
|
|
|
|
|
|
|
// now upload
|
2017-01-07 03:13:31 +00:00
|
|
|
uploadQueue := tq.NewTransferQueue(tq.Upload, manifest, "origin", tq.WithProgress(meter))
|
2015-11-24 16:37:09 +00:00
|
|
|
for _, f := range outputs[0].Files {
|
|
|
|
oidsExist = append(oidsExist, TestObject{Oid: f.Oid, Size: f.Size})
|
|
|
|
|
2017-10-25 18:20:25 +00:00
|
|
|
t, err := uploadTransfer(repo.Filesystem(), f.Oid, "Test file")
|
2015-11-24 16:37:09 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
2016-12-15 20:49:04 +00:00
|
|
|
uploadQueue.Add(t.Name, t.Path, t.Oid, t.Size)
|
2015-11-24 16:37:09 +00:00
|
|
|
}
|
|
|
|
uploadQueue.Wait()
|
|
|
|
|
|
|
|
for _, err := range uploadQueue.Errors() {
|
2016-08-18 20:20:33 +00:00
|
|
|
if errors.IsFatalError(err) {
|
2015-11-24 16:37:09 +00:00
|
|
|
exit("Fatal error setting up test data: %s", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Generate SHAs for missing files, random but repeatable
|
|
|
|
// No actual file content needed for these
|
2015-11-24 11:55:33 +00:00
|
|
|
rand.Seed(int64(oidCount))
|
|
|
|
runningSha := sha256.New()
|
|
|
|
for i := 0; i < oidCount; i++ {
|
|
|
|
runningSha.Write([]byte{byte(rand.Intn(256))})
|
|
|
|
oid := hex.EncodeToString(runningSha.Sum(nil))
|
2015-11-24 14:26:00 +00:00
|
|
|
sz := int64(rand.Intn(200)) + 50
|
|
|
|
oidsMissing = append(oidsMissing, TestObject{Oid: oid, Size: sz})
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
2015-11-24 16:37:09 +00:00
|
|
|
return oidsExist, oidsMissing, nil
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
|
|
|
|
2015-11-25 14:41:31 +00:00
|
|
|
func saveTestOids(filename string, objs []TestObject) {
|
|
|
|
f, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
|
|
|
if err != nil {
|
|
|
|
exit("Error opening file %s", filename)
|
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
for _, o := range objs {
|
|
|
|
f.WriteString(fmt.Sprintf("%s %d\n", o.Oid, o.Size))
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2017-01-07 03:13:31 +00:00
|
|
|
func runTests(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) bool {
|
2015-11-26 11:53:52 +00:00
|
|
|
ok := true
|
2015-11-24 11:55:33 +00:00
|
|
|
fmt.Printf("Running %d tests...\n", len(tests))
|
|
|
|
for _, t := range tests {
|
2017-01-07 03:13:31 +00:00
|
|
|
err := runTest(t, manifest, oidsExist, oidsMissing)
|
2015-11-26 11:53:52 +00:00
|
|
|
if err != nil {
|
|
|
|
ok = false
|
|
|
|
}
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
2015-11-26 11:53:52 +00:00
|
|
|
return ok
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
|
|
|
|
2017-01-07 03:13:31 +00:00
|
|
|
func runTest(t ServerTest, manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error {
|
2015-11-24 11:55:33 +00:00
|
|
|
const linelen = 70
|
|
|
|
line := t.Name
|
|
|
|
if len(line) > linelen {
|
|
|
|
line = line[:linelen]
|
|
|
|
} else if len(line) < linelen {
|
|
|
|
line = fmt.Sprintf("%s%s", line, strings.Repeat(" ", linelen-len(line)))
|
|
|
|
}
|
|
|
|
fmt.Printf("%s...\r", line)
|
|
|
|
|
2017-01-07 03:13:31 +00:00
|
|
|
err := t.F(manifest, oidsExist, oidsMissing)
|
2015-11-24 11:55:33 +00:00
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("%s FAILED\n", line)
|
|
|
|
fmt.Println(err.Error())
|
|
|
|
} else {
|
|
|
|
fmt.Printf("%s OK\n", line)
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Exit prints a formatted message and exits.
|
|
|
|
func exit(format string, args ...interface{}) {
|
|
|
|
fmt.Fprintf(os.Stderr, format, args...)
|
|
|
|
os.Exit(2)
|
|
|
|
}
|
|
|
|
|
2017-01-07 03:13:31 +00:00
|
|
|
func addTest(name string, f func(manifest *tq.Manifest, oidsExist, oidsMissing []TestObject) error) {
|
2015-11-25 11:57:00 +00:00
|
|
|
tests = append(tests, ServerTest{Name: name, F: f})
|
|
|
|
}
|
|
|
|
|
2017-01-07 03:13:31 +00:00
|
|
|
func callBatchApi(manifest *tq.Manifest, dir tq.Direction, objs []TestObject) ([]*tq.Transfer, error) {
|
|
|
|
apiobjs := make([]*tq.Transfer, 0, len(objs))
|
2015-11-25 14:41:31 +00:00
|
|
|
for _, o := range objs {
|
2017-01-07 03:13:31 +00:00
|
|
|
apiobjs = append(apiobjs, &tq.Transfer{Oid: o.Oid, Size: o.Size})
|
2015-11-25 14:41:31 +00:00
|
|
|
}
|
2017-01-07 03:13:31 +00:00
|
|
|
|
2017-01-09 20:00:59 +00:00
|
|
|
bres, err := tq.Batch(manifest, dir, "origin", apiobjs)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return bres.Objects, nil
|
2015-11-25 14:41:31 +00:00
|
|
|
}
|
|
|
|
|
2015-11-26 11:46:13 +00:00
|
|
|
// Combine 2 slices into one by "randomly" interleaving
|
|
|
|
// Not actually random, same sequence each time so repeatable
|
|
|
|
func interleaveTestData(slice1, slice2 []TestObject) []TestObject {
|
|
|
|
// Predictable sequence, mixin existing & missing semi-randomly
|
|
|
|
rand.Seed(21)
|
|
|
|
count := len(slice1) + len(slice2)
|
|
|
|
ret := make([]TestObject, 0, count)
|
|
|
|
slice1Idx := 0
|
|
|
|
slice2Idx := 0
|
|
|
|
for left := count; left > 0; {
|
|
|
|
for i := rand.Intn(3) + 1; slice1Idx < len(slice1) && i > 0; i-- {
|
|
|
|
obj := slice1[slice1Idx]
|
|
|
|
ret = append(ret, obj)
|
|
|
|
slice1Idx++
|
|
|
|
left--
|
|
|
|
}
|
|
|
|
for i := rand.Intn(3) + 1; slice2Idx < len(slice2) && i > 0; i-- {
|
|
|
|
obj := slice2[slice2Idx]
|
|
|
|
ret = append(ret, obj)
|
|
|
|
slice2Idx++
|
|
|
|
left--
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ret
|
|
|
|
}
|
|
|
|
|
2017-10-25 18:20:25 +00:00
|
|
|
func uploadTransfer(fs *fs.Filesystem, oid, filename string) (*tq.Transfer, error) {
|
|
|
|
localMediaPath, err := fs.ObjectPath(oid)
|
2016-12-15 20:49:04 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrapf(err, "Error uploading file %s (%s)", filename, oid)
|
|
|
|
}
|
|
|
|
|
|
|
|
fi, err := os.Stat(localMediaPath)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrapf(err, "Error uploading file %s (%s)", filename, oid)
|
|
|
|
}
|
|
|
|
|
|
|
|
return &tq.Transfer{
|
|
|
|
Name: filename,
|
|
|
|
Path: localMediaPath,
|
|
|
|
Oid: oid,
|
|
|
|
Size: fi.Size(),
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2015-11-24 11:55:33 +00:00
|
|
|
func init() {
|
|
|
|
RootCmd.Flags().StringVarP(&apiUrl, "url", "u", "", "URL of the API (must supply this or --clone)")
|
|
|
|
RootCmd.Flags().StringVarP(&cloneUrl, "clone", "c", "", "Clone URL from which to find API (must supply this or --url)")
|
2015-11-25 14:41:31 +00:00
|
|
|
RootCmd.Flags().StringVarP(&savePrefix, "save", "s", "", "Saves generated data to <prefix>_exists|missing for subsequent use")
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|