2015-11-24 11:55:33 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bufio"
|
|
|
|
"crypto/sha256"
|
|
|
|
"encoding/hex"
|
|
|
|
"fmt"
|
|
|
|
"math/rand"
|
|
|
|
"os"
|
2015-11-24 14:26:00 +00:00
|
|
|
"strconv"
|
2015-11-24 11:55:33 +00:00
|
|
|
"strings"
|
|
|
|
|
2016-05-16 11:31:23 +00:00
|
|
|
"github.com/github/git-lfs/api"
|
2016-05-13 16:38:06 +00:00
|
|
|
"github.com/github/git-lfs/config"
|
2016-05-16 14:06:02 +00:00
|
|
|
"github.com/github/git-lfs/errutil"
|
2015-11-24 11:55:33 +00:00
|
|
|
"github.com/github/git-lfs/lfs"
|
2016-05-13 16:38:06 +00:00
|
|
|
"github.com/github/git-lfs/test"
|
2016-05-23 18:02:27 +00:00
|
|
|
"github.com/spf13/cobra"
|
2015-11-24 11:55:33 +00:00
|
|
|
)
|
|
|
|
|
2015-11-24 14:26:00 +00:00
|
|
|
type TestObject struct {
|
|
|
|
Oid string
|
|
|
|
Size int64
|
|
|
|
}
|
|
|
|
|
2015-11-24 11:55:33 +00:00
|
|
|
type ServerTest struct {
|
|
|
|
Name string
|
2015-11-24 14:26:00 +00:00
|
|
|
F func(oidsExist, oidsMissing []TestObject) error
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
|
|
|
RootCmd = &cobra.Command{
|
|
|
|
Use: "git-lfs-test-server-api [--url=<apiurl> | --clone=<cloneurl>] [<oid-exists-file> <oid-missing-file>]",
|
|
|
|
Short: "Test a Git LFS API server for compliance",
|
|
|
|
Run: testServerApi,
|
|
|
|
}
|
2015-11-25 14:41:31 +00:00
|
|
|
apiUrl string
|
|
|
|
cloneUrl string
|
|
|
|
savePrefix string
|
2015-11-24 11:55:33 +00:00
|
|
|
|
|
|
|
tests []ServerTest
|
|
|
|
)
|
|
|
|
|
|
|
|
func main() {
|
|
|
|
RootCmd.Execute()
|
|
|
|
}
|
|
|
|
|
|
|
|
func testServerApi(cmd *cobra.Command, args []string) {
|
|
|
|
|
|
|
|
if (len(apiUrl) == 0 && len(cloneUrl) == 0) ||
|
|
|
|
(len(apiUrl) != 0 && len(cloneUrl) != 0) {
|
|
|
|
exit("Must supply either --url or --clone (and not both)")
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(args) != 0 && len(args) != 2 {
|
|
|
|
exit("Must supply either no file arguments or both the exists AND missing file")
|
|
|
|
}
|
|
|
|
|
2015-11-25 14:41:31 +00:00
|
|
|
if len(args) != 0 && len(savePrefix) > 0 {
|
|
|
|
exit("Cannot combine input files and --save option")
|
|
|
|
}
|
|
|
|
|
2015-11-25 10:29:54 +00:00
|
|
|
// Force loading of config before we alter it
|
2016-05-13 16:44:58 +00:00
|
|
|
config.Config.AllGitConfig()
|
2015-11-25 10:29:54 +00:00
|
|
|
|
2015-11-24 13:05:47 +00:00
|
|
|
// Configure the endpoint manually
|
2016-05-13 16:38:06 +00:00
|
|
|
var endp config.Endpoint
|
2015-11-24 11:55:33 +00:00
|
|
|
if len(cloneUrl) > 0 {
|
2016-05-13 16:38:06 +00:00
|
|
|
endp = config.NewEndpointFromCloneURL(cloneUrl)
|
2015-11-24 11:55:33 +00:00
|
|
|
} else {
|
2016-05-13 16:38:06 +00:00
|
|
|
endp = config.NewEndpoint(apiUrl)
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
2016-05-13 16:44:58 +00:00
|
|
|
config.Config.SetManualEndpoint(endp)
|
2015-11-24 11:55:33 +00:00
|
|
|
|
2015-11-24 14:26:00 +00:00
|
|
|
var oidsExist, oidsMissing []TestObject
|
2015-11-24 11:55:33 +00:00
|
|
|
if len(args) >= 2 {
|
|
|
|
fmt.Printf("Reading test data from files (no server content changes)\n")
|
|
|
|
oidsExist = readTestOids(args[0])
|
|
|
|
oidsMissing = readTestOids(args[1])
|
|
|
|
} else {
|
2015-11-25 10:35:01 +00:00
|
|
|
fmt.Printf("Creating test data (will upload to server)\n")
|
2015-11-24 16:37:09 +00:00
|
|
|
var err error
|
|
|
|
oidsExist, oidsMissing, err = buildTestData()
|
2015-11-24 11:55:33 +00:00
|
|
|
if err != nil {
|
|
|
|
exit("Failed to set up test data, aborting")
|
|
|
|
}
|
2015-11-25 14:41:31 +00:00
|
|
|
if len(savePrefix) > 0 {
|
|
|
|
existFile := savePrefix + "_exists"
|
|
|
|
missingFile := savePrefix + "_missing"
|
|
|
|
saveTestOids(existFile, oidsExist)
|
|
|
|
saveTestOids(missingFile, oidsMissing)
|
|
|
|
fmt.Printf("Wrote test to %s, %s for future use\n", existFile, missingFile)
|
|
|
|
}
|
|
|
|
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
|
|
|
|
2015-11-26 11:53:52 +00:00
|
|
|
ok := runTests(oidsExist, oidsMissing)
|
|
|
|
if !ok {
|
|
|
|
exit("One or more tests failed, see above")
|
|
|
|
}
|
|
|
|
fmt.Println("All tests passed")
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
|
|
|
|
2015-11-24 14:26:00 +00:00
|
|
|
func readTestOids(filename string) []TestObject {
|
2015-11-24 11:55:33 +00:00
|
|
|
f, err := os.OpenFile(filename, os.O_RDONLY, 0644)
|
|
|
|
if err != nil {
|
|
|
|
exit("Error opening file %s", filename)
|
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
2015-11-24 14:26:00 +00:00
|
|
|
var ret []TestObject
|
2015-11-24 11:55:33 +00:00
|
|
|
rdr := bufio.NewReader(f)
|
|
|
|
line, err := rdr.ReadString('\n')
|
|
|
|
for err == nil {
|
2015-11-24 14:26:00 +00:00
|
|
|
fields := strings.Fields(strings.TrimSpace(line))
|
|
|
|
if len(fields) == 2 {
|
|
|
|
sz, _ := strconv.ParseInt(fields[1], 10, 64)
|
|
|
|
ret = append(ret, TestObject{Oid: fields[0], Size: sz})
|
|
|
|
}
|
|
|
|
|
2015-11-24 11:55:33 +00:00
|
|
|
line, err = rdr.ReadString('\n')
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret
|
|
|
|
}
|
|
|
|
|
2015-11-24 16:37:09 +00:00
|
|
|
type testDataCallback struct{}
|
|
|
|
|
|
|
|
func (*testDataCallback) Fatalf(format string, args ...interface{}) {
|
|
|
|
exit(format, args...)
|
|
|
|
}
|
|
|
|
func (*testDataCallback) Errorf(format string, args ...interface{}) {
|
|
|
|
fmt.Printf(format, args...)
|
|
|
|
}
|
|
|
|
|
|
|
|
func buildTestData() (oidsExist, oidsMissing []TestObject, err error) {
|
2015-11-24 11:55:33 +00:00
|
|
|
const oidCount = 50
|
2015-11-24 14:26:00 +00:00
|
|
|
oidsExist = make([]TestObject, 0, oidCount)
|
|
|
|
oidsMissing = make([]TestObject, 0, oidCount)
|
2015-11-24 11:55:33 +00:00
|
|
|
|
2015-11-24 16:37:09 +00:00
|
|
|
// Build test data for existing files & upload
|
|
|
|
// Use test repo for this to simplify the process of making sure data matches oid
|
|
|
|
// We're not performing a real test at this point (although an upload fail will break it)
|
|
|
|
var callback testDataCallback
|
|
|
|
repo := test.NewRepo(&callback)
|
|
|
|
repo.Pushd()
|
|
|
|
defer repo.Cleanup()
|
|
|
|
// just one commit
|
|
|
|
commit := test.CommitInput{CommitterName: "A N Other", CommitterEmail: "noone@somewhere.com"}
|
|
|
|
var totalSize int64
|
|
|
|
for i := 0; i < oidCount; i++ {
|
|
|
|
filename := fmt.Sprintf("file%d.dat", i)
|
|
|
|
sz := int64(rand.Intn(200)) + 50
|
|
|
|
commit.Files = append(commit.Files, &test.FileInput{Filename: filename, Size: sz})
|
|
|
|
totalSize += sz
|
|
|
|
}
|
|
|
|
outputs := repo.AddCommits([]*test.CommitInput{&commit})
|
|
|
|
|
|
|
|
// now upload
|
|
|
|
uploadQueue := lfs.NewUploadQueue(len(oidsExist), totalSize, false)
|
|
|
|
for _, f := range outputs[0].Files {
|
|
|
|
oidsExist = append(oidsExist, TestObject{Oid: f.Oid, Size: f.Size})
|
|
|
|
|
|
|
|
u, err := lfs.NewUploadable(f.Oid, "Test file")
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
|
|
|
uploadQueue.Add(u)
|
|
|
|
}
|
|
|
|
uploadQueue.Wait()
|
|
|
|
|
|
|
|
for _, err := range uploadQueue.Errors() {
|
2016-05-16 14:06:02 +00:00
|
|
|
if errutil.IsFatalError(err) {
|
2015-11-24 16:37:09 +00:00
|
|
|
exit("Fatal error setting up test data: %s", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Generate SHAs for missing files, random but repeatable
|
|
|
|
// No actual file content needed for these
|
2015-11-24 11:55:33 +00:00
|
|
|
rand.Seed(int64(oidCount))
|
|
|
|
runningSha := sha256.New()
|
|
|
|
for i := 0; i < oidCount; i++ {
|
|
|
|
runningSha.Write([]byte{byte(rand.Intn(256))})
|
|
|
|
oid := hex.EncodeToString(runningSha.Sum(nil))
|
2015-11-24 14:26:00 +00:00
|
|
|
sz := int64(rand.Intn(200)) + 50
|
|
|
|
oidsMissing = append(oidsMissing, TestObject{Oid: oid, Size: sz})
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
2015-11-24 16:37:09 +00:00
|
|
|
return oidsExist, oidsMissing, nil
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
|
|
|
|
2015-11-25 14:41:31 +00:00
|
|
|
func saveTestOids(filename string, objs []TestObject) {
|
|
|
|
f, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
|
|
|
if err != nil {
|
|
|
|
exit("Error opening file %s", filename)
|
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
for _, o := range objs {
|
|
|
|
f.WriteString(fmt.Sprintf("%s %d\n", o.Oid, o.Size))
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2015-11-26 11:53:52 +00:00
|
|
|
func runTests(oidsExist, oidsMissing []TestObject) bool {
|
2015-11-24 11:55:33 +00:00
|
|
|
|
2015-11-26 11:53:52 +00:00
|
|
|
ok := true
|
2015-11-24 11:55:33 +00:00
|
|
|
fmt.Printf("Running %d tests...\n", len(tests))
|
|
|
|
for _, t := range tests {
|
2015-11-26 11:53:52 +00:00
|
|
|
err := runTest(t, oidsExist, oidsMissing)
|
|
|
|
if err != nil {
|
|
|
|
ok = false
|
|
|
|
}
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
2015-11-26 11:53:52 +00:00
|
|
|
return ok
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|
|
|
|
|
2015-11-24 14:26:00 +00:00
|
|
|
func runTest(t ServerTest, oidsExist, oidsMissing []TestObject) error {
|
2015-11-24 11:55:33 +00:00
|
|
|
const linelen = 70
|
|
|
|
line := t.Name
|
|
|
|
if len(line) > linelen {
|
|
|
|
line = line[:linelen]
|
|
|
|
} else if len(line) < linelen {
|
|
|
|
line = fmt.Sprintf("%s%s", line, strings.Repeat(" ", linelen-len(line)))
|
|
|
|
}
|
|
|
|
fmt.Printf("%s...\r", line)
|
|
|
|
|
2015-11-24 13:05:47 +00:00
|
|
|
err := t.F(oidsExist, oidsMissing)
|
2015-11-24 11:55:33 +00:00
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("%s FAILED\n", line)
|
|
|
|
fmt.Println(err.Error())
|
|
|
|
} else {
|
|
|
|
fmt.Printf("%s OK\n", line)
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Exit prints a formatted message and exits.
|
|
|
|
func exit(format string, args ...interface{}) {
|
|
|
|
fmt.Fprintf(os.Stderr, format, args...)
|
|
|
|
os.Exit(2)
|
|
|
|
}
|
|
|
|
|
2015-11-25 11:57:00 +00:00
|
|
|
func addTest(name string, f func(oidsExist, oidsMissing []TestObject) error) {
|
|
|
|
tests = append(tests, ServerTest{Name: name, F: f})
|
|
|
|
}
|
|
|
|
|
2016-05-16 11:31:23 +00:00
|
|
|
func callBatchApi(op string, objs []TestObject) ([]*api.ObjectResource, error) {
|
2015-11-25 14:41:31 +00:00
|
|
|
|
2016-05-16 11:31:23 +00:00
|
|
|
apiobjs := make([]*api.ObjectResource, 0, len(objs))
|
2015-11-25 14:41:31 +00:00
|
|
|
for _, o := range objs {
|
2016-05-16 11:31:23 +00:00
|
|
|
apiobjs = append(apiobjs, &api.ObjectResource{Oid: o.Oid, Size: o.Size})
|
2015-11-25 14:41:31 +00:00
|
|
|
}
|
2016-06-02 11:11:45 +00:00
|
|
|
o, _, err := api.Batch(apiobjs, op, []string{"basic"})
|
2016-06-01 16:33:01 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2016-06-02 11:11:45 +00:00
|
|
|
return o, nil
|
2015-11-25 14:41:31 +00:00
|
|
|
}
|
|
|
|
|
2015-11-26 11:46:13 +00:00
|
|
|
// Combine 2 slices into one by "randomly" interleaving
|
|
|
|
// Not actually random, same sequence each time so repeatable
|
|
|
|
func interleaveTestData(slice1, slice2 []TestObject) []TestObject {
|
|
|
|
// Predictable sequence, mixin existing & missing semi-randomly
|
|
|
|
rand.Seed(21)
|
|
|
|
count := len(slice1) + len(slice2)
|
|
|
|
ret := make([]TestObject, 0, count)
|
|
|
|
slice1Idx := 0
|
|
|
|
slice2Idx := 0
|
|
|
|
for left := count; left > 0; {
|
|
|
|
for i := rand.Intn(3) + 1; slice1Idx < len(slice1) && i > 0; i-- {
|
|
|
|
obj := slice1[slice1Idx]
|
|
|
|
ret = append(ret, obj)
|
|
|
|
slice1Idx++
|
|
|
|
left--
|
|
|
|
}
|
|
|
|
for i := rand.Intn(3) + 1; slice2Idx < len(slice2) && i > 0; i-- {
|
|
|
|
obj := slice2[slice2Idx]
|
|
|
|
ret = append(ret, obj)
|
|
|
|
slice2Idx++
|
|
|
|
left--
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ret
|
|
|
|
}
|
|
|
|
|
2015-11-24 11:55:33 +00:00
|
|
|
func init() {
|
|
|
|
RootCmd.Flags().StringVarP(&apiUrl, "url", "u", "", "URL of the API (must supply this or --clone)")
|
|
|
|
RootCmd.Flags().StringVarP(&cloneUrl, "clone", "c", "", "Clone URL from which to find API (must supply this or --url)")
|
2015-11-25 14:41:31 +00:00
|
|
|
RootCmd.Flags().StringVarP(&savePrefix, "save", "s", "", "Saves generated data to <prefix>_exists|missing for subsequent use")
|
2015-11-24 11:55:33 +00:00
|
|
|
}
|