lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into transfers-p5-ext-process

Browse Source
This commit is contained in:
Taylor Blau 2016-07-17 15:18:05 -07:00 committed by GitHub
commit 51c3320760
11 changed files with 122 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -15,8 +15,6 @@ env:
matrix:
fast_finish: true
allow_failures:
- os: osx
include:
- env: git-from-source
os: linux

8
api/schema/schema_validator.go
View File

@ -25,10 +25,12 @@ func NewSchemaValidator(t *testing.T, schemaName string, got interface{}) *Schem
t.Fatal(err)
}
// Platform compatibility: use "/" separators always for file://
dir = filepath.ToSlash(dir)
schema := gojsonschema.NewReferenceLoader(fmt.Sprintf(
"file:///%s",
filepath.Join(dir, "schema/", schemaName),
))
"file:///%s/schema/%s", dir, schemaName),
)
marshalled, err := json.Marshal(got)
if err != nil {

7
script/integration
View File

@ -41,9 +41,4 @@ fi
setup
parallel=${GIT_LFS_TEST_MAXPROCS:-4}
echo "Running this maxprocs=$parallel"
echo
GO15VENDOREXPERIMENT=1 GIT_LFS_TEST_MAXPROCS=$parallel GIT_LFS_TEST_DIR="$GIT_LFS_TEST_DIR" SHUTDOWN_LFS="no" go run script/*.go -cmd integration "$@"
GO15VENDOREXPERIMENT=1 GIT_LFS_TEST_MAXPROCS=$GIT_LFS_TEST_MAXPROCS GIT_LFS_TEST_DIR="$GIT_LFS_TEST_DIR" SHUTDOWN_LFS="no" go run script/*.go -cmd integration "$@"

9
script/integration.go
View File

@ -9,6 +9,7 @@ import (
"path/filepath"
"regexp"
"runtime"
"strconv"
"strings"
"sync"
"time"
@ -29,10 +30,12 @@ func mainIntegration() {
setBash()
if maxprocs < 1 {
maxprocs = 1
if max, _ := strconv.Atoi(os.Getenv("GIT_LFS_TEST_MAXPROCS")); max > 0 {
maxprocs = max
}
fmt.Println("Running this maxprocs", maxprocs)
files := testFiles()
if len(files) == 0 {
@ -93,8 +96,6 @@ func runTest(output chan string, testname string) {
cmd.Process.Kill()
return
}
sendTestOutput(output, testname, buf, nil)
}
func sendTestOutput(output chan string, testname string, buf *bytes.Buffer, err error) {

3
script/test
View File

@ -11,7 +11,10 @@ else
GO15VENDOREXPERIMENT=1 go test \
$(GO15VENDOREXPERIMENT=1 go list ./... \
| grep -v "github.com/olekukonko/ts" \
| grep -v "github.com/xeipuuv/gojsonreference" \
| grep -v "github.com/xeipuuv/gojsonschema" \
| grep -v "github.com/technoweenie/go-contentaddressable" \
| grep -v "github.com/kr/pty" \
| grep -v "github.com/kr/text" \
)
fi

115
test/cmd/lfstest-gitserver.go
View File

@ -71,15 +71,20 @@ func main() {
mux.HandleFunc("/locks", locksHandler)
mux.HandleFunc("/locks/", locksHandler)
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
id, ok := reqId(w)
if !ok {
return
}
if strings.Contains(r.URL.Path, "/info/lfs") {
if !skipIfBadAuth(w, r) {
lfsHandler(w, r)
if !skipIfBadAuth(w, r, id) {
lfsHandler(w, r, id)
}
return
}
log.Printf("git http-backend %s %s\n", r.Method, r.URL)
debug(id, "git http-backend %s %s", r.Method, r.URL)
gitHandler(w, r)
})
@ -96,11 +101,11 @@ func main() {
certname := writeTestStateFile(pembytes, "LFSTEST_CERT", "lfstest-gitserver-cert")
defer os.RemoveAll(certname)
log.Println(server.URL)
log.Println(serverTLS.URL)
debug("init", "server url: %s", server.URL)
debug("init", "server tls url: %s", serverTLS.URL)
<-stopch
log.Println("git server done")
debug("init", "git server done")
}
// writeTestStateFile writes contents to either the file referenced by the
@ -139,7 +144,7 @@ type lfsError struct {
}
// handles any requests with "{name}.server.git/info/lfs" in the path
func lfsHandler(w http.ResponseWriter, r *http.Request) {
func lfsHandler(w http.ResponseWriter, r *http.Request, id string) {
repo, err := repoFromLfsUrl(r.URL.Path)
if err != nil {
w.WriteHeader(500)
@ -147,17 +152,17 @@ func lfsHandler(w http.ResponseWriter, r *http.Request) {
return
}
log.Printf("git lfs %s %s repo: %s\n", r.Method, r.URL, repo)
debug(id, "git lfs %s %s repo: %s", r.Method, r.URL, repo)
w.Header().Set("Content-Type", "application/vnd.git-lfs+json")
switch r.Method {
case "POST":
if strings.HasSuffix(r.URL.String(), "batch") {
lfsBatchHandler(w, r, repo)
lfsBatchHandler(w, r, id, repo)
} else {
lfsPostHandler(w, r, repo)
lfsPostHandler(w, r, id, repo)
}
case "GET":
lfsGetHandler(w, r, repo)
lfsGetHandler(w, r, id, repo)
default:
w.WriteHeader(405)
}
@ -167,7 +172,7 @@ func lfsUrl(repo, oid string) string {
return server.URL + "/storage/" + oid + "?r=" + repo
}
func lfsPostHandler(w http.ResponseWriter, r *http.Request, repo string) {
func lfsPostHandler(w http.ResponseWriter, r *http.Request, id, repo string) {
buf := &bytes.Buffer{}
tee := io.TeeReader(r.Body, buf)
obj := &lfsObject{}
@ -175,10 +180,10 @@ func lfsPostHandler(w http.ResponseWriter, r *http.Request, repo string) {
io.Copy(ioutil.Discard, r.Body)
r.Body.Close()
log.Println("REQUEST")
log.Println(buf.String())
log.Printf("OID: %s\n", obj.Oid)
log.Printf("Size: %d\n", obj.Size)
debug(id, "REQUEST")
debug(id, buf.String())
debug(id, "OID: %s", obj.Oid)
debug(id, "Size: %d", obj.Size)
if err != nil {
log.Fatal(err)
@ -222,21 +227,21 @@ func lfsPostHandler(w http.ResponseWriter, r *http.Request, repo string) {
log.Fatal(err)
}
log.Println("RESPONSE: 202")
log.Println(string(by))
debug(id, "RESPONSE: 202")
debug(id, string(by))
w.WriteHeader(202)
w.Write(by)
}
func lfsGetHandler(w http.ResponseWriter, r *http.Request, repo string) {
func lfsGetHandler(w http.ResponseWriter, r *http.Request, id, repo string) {
parts := strings.Split(r.URL.Path, "/")
oid := parts[len(parts)-1]
// Support delete for testing
if len(parts) > 1 && parts[len(parts)-2] == "delete" {
largeObjects.Delete(repo, oid)
log.Println("DELETE:", oid)
debug(id, "DELETE:", oid)
w.WriteHeader(200)
return
}
@ -262,14 +267,14 @@ func lfsGetHandler(w http.ResponseWriter, r *http.Request, repo string) {
log.Fatal(err)
}
log.Println("RESPONSE: 200")
log.Println(string(by))
debug(id, "RESPONSE: 200")
debug(id, string(by))
w.WriteHeader(200)
w.Write(by)
}
func lfsBatchHandler(w http.ResponseWriter, r *http.Request, repo string) {
func lfsBatchHandler(w http.ResponseWriter, r *http.Request, id, repo string) {
if repo == "batchunsupported" {
w.WriteHeader(404)
return
@ -309,8 +314,8 @@ func lfsBatchHandler(w http.ResponseWriter, r *http.Request, repo string) {
io.Copy(ioutil.Discard, r.Body)
r.Body.Close()
log.Println("REQUEST")
log.Println(buf.String())
debug(id, "REQUEST")
debug(id, buf.String())
if err != nil {
log.Fatal(err)
@ -409,8 +414,8 @@ func lfsBatchHandler(w http.ResponseWriter, r *http.Request, repo string) {
log.Fatal(err)
}
log.Println("RESPONSE: 200")
log.Println(string(by))
debug(id, "RESPONSE: 200")
debug(id, string(by))
w.WriteHeader(200)
w.Write(by)
@ -448,6 +453,11 @@ var tusStorageAttempts = 0
// handles any /storage/{oid} requests
func storageHandler(w http.ResponseWriter, r *http.Request) {
id, ok := reqId(w)
if !ok {
return
}
repo := r.URL.Query().Get("r")
parts := strings.Split(r.URL.Path, "/")
oid := parts[len(parts)-1]
@ -455,7 +465,7 @@ func storageHandler(w http.ResponseWriter, r *http.Request) {
return
}
log.Printf("storage %s %s repo: %s\n", r.Method, oid, repo)
debug(id, "storage %s %s repo: %s", r.Method, oid, repo)
switch r.Method {
case "PUT":
switch oidHandlers[oid] {
@ -485,7 +495,7 @@ func storageHandler(w http.ResponseWriter, r *http.Request) {
}
}
if !valid {
log.Fatal("Chunked transfer encoding expected")
debug(id, "Chunked transfer encoding expected")
}
}
@ -550,7 +560,7 @@ func storageHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(404)
case "HEAD":
// tus.io
if !validateTusHeaders(r) {
if !validateTusHeaders(r, id) {
w.WriteHeader(400)
return
}
@ -564,7 +574,7 @@ func storageHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
case "PATCH":
// tus.io
if !validateTusHeaders(r) {
if !validateTusHeaders(r, id) {
w.WriteHeader(400)
return
}
@ -595,7 +605,7 @@ func storageHandler(w http.ResponseWriter, r *http.Request) {
return
}
largeObjects.DeleteIncomplete(repo, oid)
log.Printf("Resuming upload of %v at byte %d", oid, offset)
debug(id, "Resuming upload of %v at byte %d", oid, offset)
}
// As a test, we intentionally break the upload from byte 0 by only
@ -620,7 +630,7 @@ func storageHandler(w http.ResponseWriter, r *http.Request) {
if copyErr != nil {
b := buf.Bytes()
if len(b) > 0 {
log.Printf("Incomplete upload of %v, %d bytes", oid, len(b))
debug(id, "Incomplete upload of %v, %d bytes", oid, len(b))
largeObjects.SetIncomplete(repo, oid, b)
}
w.WriteHeader(500)
@ -643,13 +653,12 @@ func storageHandler(w http.ResponseWriter, r *http.Request) {
}
}
func validateTusHeaders(r *http.Request) bool {
func validateTusHeaders(r *http.Request, id string) bool {
if len(r.Header.Get("Tus-Resumable")) == 0 {
log.Fatal("Missing Tus-Resumable header in request")
debug(id, "Missing Tus-Resumable header in request")
return false
}
return true
}
func gitHandler(w http.ResponseWriter, r *http.Request) {
@ -697,6 +706,11 @@ func gitHandler(w http.ResponseWriter, r *http.Request) {
}
func redirect307Handler(w http.ResponseWriter, r *http.Request) {
id, ok := reqId(w)
if !ok {
return
}
// Send a redirect to info/lfs
// Make it either absolute or relative depending on subpath
parts := strings.Split(r.URL.Path, "/")
@ -707,7 +721,7 @@ func redirect307Handler(w http.ResponseWriter, r *http.Request) {
} else if parts[2] == "abs" {
redirectTo = server.URL + "/" + strings.Join(parts[3:], "/")
} else {
log.Fatal(fmt.Errorf("Invalid URL for redirect: %v", r.URL))
debug(id, "Invalid URL for redirect: %v", r.URL)
w.WriteHeader(404)
return
}
@ -1087,7 +1101,7 @@ func extractAuth(auth string) (string, string, error) {
return "", "", nil
}
func skipIfBadAuth(w http.ResponseWriter, r *http.Request) bool {
func skipIfBadAuth(w http.ResponseWriter, r *http.Request, id string) bool {
auth := r.Header.Get("Authorization")
if auth == "" {
w.WriteHeader(401)
@ -1097,7 +1111,7 @@ func skipIfBadAuth(w http.ResponseWriter, r *http.Request) bool {
user, pass, err := extractAuth(auth)
if err != nil {
w.WriteHeader(403)
log.Printf("Error decoding auth: %s\n", err)
debug(id, "Error decoding auth: %s", err)
return true
}
@ -1112,11 +1126,11 @@ func skipIfBadAuth(w http.ResponseWriter, r *http.Request) bool {
if strings.HasPrefix(r.URL.Path, "/"+pass) {
return false
}
log.Printf("auth attempt against: %q", r.URL.Path)
debug(id, "auth attempt against: %q", r.URL.Path)
}
w.WriteHeader(403)
log.Printf("Bad auth: %q\n", auth)
debug(id, "Bad auth: %q", auth)
return true
}
@ -1128,3 +1142,22 @@ func init() {
oidHandlers[hex.EncodeToString(h.Sum(nil))] = content
}
}
func debug(reqid, msg string, args ...interface{}) {
fullargs := make([]interface{}, len(args)+1)
fullargs[0] = reqid
for i, a := range args {
fullargs[i+1] = a
}
log.Printf("[%s] "+msg+"\n", fullargs...)
}
func reqId(w http.ResponseWriter) (string, bool) {
b := make([]byte, 16)
_, err := rand.Read(b)
if err != nil {
http.Error(w, "error generating id: "+err.Error(), 500)
return "", false
}
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), true
}

10
test/test-credentials-no-prompt.sh
View File

@ -4,13 +4,6 @@
# these tests rely on GIT_TERMINAL_PROMPT to test properly
ensure_git_version_isnt $VERSION_LOWER "2.3.0"
# if there is a system cred helper we can't run this test
# can't disable without changing state outside test & probably don't have permission
# this is common on OS X with certain versions of Git installed, default cred helper
if [[ "$(git config --system credential.helper)" -ne "" ]]; then
echo "skip: $0 (system cred helper we can't disable)"
exit
fi
# if there is a system cred helper we can't run this test
# can't disable without changing state outside test & probably don't have permission
@ -34,12 +27,11 @@ begin_test "attempt private access without credential helper"
git add .gitattributes
git commit -m "initial commit"
git config credential.usehttppath true
git config --global credential.helper lfsnoop
git config credential.helper lfsnoop
git config -l
GIT_TERMINAL_PROMPT=0 git push origin master 2>&1 | tee push.log
git push origin master 2>&1 | tee push.log
grep "Authorization error: $GITSERVER/$reponame" push.log ||
grep "Git credentials for $GITSERVER/$reponame not found" push.log
)

35
test/test-credentials.sh
View File

@ -2,6 +2,8 @@
. "test/testlib.sh"
ensure_git_version_isnt $VERSION_LOWER "2.3.0"
begin_test "credentials without useHttpPath, with bad path password"
(
set -e
@ -12,6 +14,7 @@ begin_test "credentials without useHttpPath, with bad path password"
printf "path:wrong" > "$CREDSDIR/127.0.0.1--$reponame"
clone_repo "$reponame" without-path
git config credential.useHttpPath false
git checkout -b without-path
git lfs track "*.dat" 2>&1 | tee track.log
@ -40,7 +43,6 @@ begin_test "credentials with useHttpPath, with wrong password"
printf "path:wrong" > "$CREDSDIR/127.0.0.1--$reponame"
clone_repo "$reponame" with-path-wrong-pass
git config credential.useHttpPath true
git checkout -b with-path-wrong-pass
git lfs track "*.dat" 2>&1 | tee track.log
@ -69,7 +71,6 @@ begin_test "credentials with useHttpPath, with correct password"
printf "path:$reponame" > "$CREDSDIR/127.0.0.1--$reponame"
clone_repo "$reponame" with-path-correct-pass
git config credential.useHttpPath true
git checkout -b with-path-correct-pass
git lfs track "*.dat" 2>&1 | tee track.log
@ -101,6 +102,21 @@ begin_test "git credential"
cd empty
git init
echo "protocol=http
host=credential-test.com
path=some/path" | GIT_TERMINAL_PROMPT=0 git credential fill > cred.log
cat cred.log
expected="protocol=http
host=credential-test.com
path=some/path
username=git
password=path"
[ "$expected" = "$(cat cred.log)" ]
git config credential.useHttpPath false
echo "protocol=http
host=credential-test.com" | GIT_TERMINAL_PROMPT=0 git credential fill > cred.log
cat cred.log
@ -122,21 +138,6 @@ username=git
password=server"
[ "$expected" = "$(cat cred.log)" ]
git config credential.useHttpPath true
echo "protocol=http
host=credential-test.com
path=some/path" | GIT_TERMINAL_PROMPT=0 git credential fill > cred.log
cat cred.log
expected="protocol=http
host=credential-test.com
path=some/path
username=git
password=path"
[ "$expected" = "$(cat cred.log)" ]
)
end_test

8
test/testenv.sh
View File

@ -60,7 +60,7 @@ if [ -z "$GIT_LFS_TEST_DIR" ]; then
GIT_LFS_TEST_DIR=$(resolve_symlink $GIT_LFS_TEST_DIR)
# cleanup either after single test or at end of integration (except on fail)
RM_GIT_LFS_TEST_DIR=yes
fi
fi
# create a temporary work space
TMPDIR=$GIT_LFS_TEST_DIR
@ -102,6 +102,7 @@ LFS_CERT_FILE="$REMOTEDIR/cert"
TESTHOME="$REMOTEDIR/home"
GIT_CONFIG_NOSYSTEM=1
GIT_TERMINAL_PROMPT=0
export CREDSDIR
@ -113,4 +114,9 @@ fi
mkdir -p "$TMPDIR"
mkdir -p "$TRASHDIR"
if [ $IS_WINDOWS == "1" ]; then
# prevent Windows OpenSSH from opening GUI prompts
SSH_ASKPASS=""
fi
. "test/testhelpers.sh"

1
test/testhelpers.sh
View File

@ -302,6 +302,7 @@ setup() {
HOME="$TESTHOME"
mkdir "$HOME"
git lfs install
git config --global credential.usehttppath true
git config --global credential.helper lfstest
git config --global user.name "Git LFS Tests"
git config --global user.email "git-lfs@example.com"

11
tools/filetools.go
View File

@ -7,6 +7,7 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"strings"
)
@ -78,11 +79,12 @@ func RenameFileCopyPermissions(srcfile, destfile string) error {
}
// CleanPaths splits the given `paths` argument by the delimiter argument, and
// then "cleans" that path according to the filepath.Clean function (see
// https://golang.org/pkg/file/filepath#Clean).
// then "cleans" that path according to the path.Clean function (see
// https://golang.org/pkg/path#Clean).
// Note always cleans to '/' path separators regardless of platform (git friendly)
func CleanPaths(paths, delim string) (cleaned []string) {
// If paths is an empty string, splitting it will yield [""], which will
// become the filepath ".". To avoid this, bail out if trimmed paths
// become the path ".". To avoid this, bail out if trimmed paths
// argument is empty.
if paths = strings.TrimSpace(paths); len(paths) == 0 {
return
@ -91,7 +93,7 @@ func CleanPaths(paths, delim string) (cleaned []string) {
for _, part := range strings.Split(paths, delim) {
part = strings.TrimSpace(part)
cleaned = append(cleaned, filepath.Clean(part))
cleaned = append(cleaned, path.Clean(part))
}
return cleaned
@ -100,6 +102,7 @@ func CleanPaths(paths, delim string) (cleaned []string) {
// CleanPathsDefault cleans the paths contained in the given `paths` argument
// delimited by the `delim`, argument. If an empty set is returned from that
// split, then the fallback argument is returned instead.
// Note always cleans to '/' path separators regardless of platform (git friendly)
func CleanPathsDefault(paths, delim string, fallback []string) []string {
cleaned := CleanPaths(paths, delim)
if len(cleaned) == 0 {