lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into pull-errors

Browse Source
This commit is contained in:
Taylor Blau 2017-05-19 10:57:19 -06:00 committed by GitHub
commit a62b4424f9
13 changed files with 473 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/man/git-lfs.1.ronn
View File

@ -41,6 +41,10 @@ commands and low level ("plumbing") commands.
Check GIT LFS files for consistency.
* git-lfs-install(1):
Install Git LFS configuration.
* git-lfs-lock(1):
Set a file as "locked" on the Git LFS server.
* git-lfs-locks(1):
List currently locked files from the Git LFS server.
* git-lfs-logs(1):
Show errors from the git-lfs command.
* git-lfs-ls-files(1):
@ -53,6 +57,8 @@ commands and low level ("plumbing") commands.
Show the status of Git LFS files in the working tree.
* git-lfs-track(1):
View or add Git LFS paths to Git attributes.
* git-lfs-unlock(1):
Remove "locked" setting for a file on the Git LFS server.
* git-lfs-untrack(1):
Remove Git LFS paths from Git Attributes.
* git-lfs-update(1):

64
filepathfilter/bench_test.go
View File

@ -10,7 +10,27 @@ import (
"github.com/git-lfs/git-lfs/tools"
)
func BenchmarkFilterIncludeWildcardOnly(b *testing.B) {
func BenchmarkFilterSimplePath(b *testing.B) {
files := benchmarkTree(b)
filter := filepathfilter.New([]string{"lfs"}, nil)
for i := 0; i < b.N; i++ {
for _, f := range files {
filter.Allows(f)
}
}
}
func BenchmarkPatternSimplePath(b *testing.B) {
files := benchmarkTree(b)
pattern := filepathfilter.NewPattern("lfs")
for i := 0; i < b.N; i++ {
for _, f := range files {
pattern.Match(f)
}
}
}
func BenchmarkFilterSimpleExtension(b *testing.B) {
files := benchmarkTree(b)
filter := filepathfilter.New([]string{"*.go"}, nil)
for i := 0; i < b.N; i++ {
@ -20,7 +40,37 @@ func BenchmarkFilterIncludeWildcardOnly(b *testing.B) {
}
}
func BenchmarkFilterIncludeDoubleAsterisk(b *testing.B) {
func BenchmarkPatternSimpleExtension(b *testing.B) {
files := benchmarkTree(b)
pattern := filepathfilter.NewPattern("*.go")
for i := 0; i < b.N; i++ {
for _, f := range files {
pattern.Match(f)
}
}
}
func BenchmarkFilterComplexExtension(b *testing.B) {
files := benchmarkTree(b)
filter := filepathfilter.New([]string{"*.travis.yml"}, nil)
for i := 0; i < b.N; i++ {
for _, f := range files {
filter.Allows(f)
}
}
}
func BenchmarkPatternComplexExtension(b *testing.B) {
files := benchmarkTree(b)
pattern := filepathfilter.NewPattern("*.travis.yml")
for i := 0; i < b.N; i++ {
for _, f := range files {
pattern.Match(f)
}
}
}
func BenchmarkFilterDoubleAsterisk(b *testing.B) {
files := benchmarkTree(b)
filter := filepathfilter.New([]string{"**/README.md"}, nil)
for i := 0; i < b.N; i++ {
@ -30,6 +80,16 @@ func BenchmarkFilterIncludeDoubleAsterisk(b *testing.B) {
}
}
func BenchmarkPatternDoubleAsterisk(b *testing.B) {
files := benchmarkTree(b)
pattern := filepathfilter.NewPattern("**/README.md")
for i := 0; i < b.N; i++ {
for _, f := range files {
pattern.Match(f)
}
}
}
var (
benchmarkFiles []string
benchmarkMu sync.Mutex

76
filepathfilter/filepathfilter.go
View File

@ -67,7 +67,13 @@ func NewPattern(rawpattern string) Pattern {
return noOpMatcher{}
}
hasPathSep := strings.Contains(cleanpattern, string(filepath.Separator))
sep := string(filepath.Separator)
hasPathSep := strings.Contains(cleanpattern, sep)
ext := filepath.Ext(cleanpattern)
plen := len(cleanpattern)
if plen > 1 && !hasPathSep && strings.HasPrefix(cleanpattern, "*") && cleanpattern[1:plen] == ext {
return &simpleExtPattern{ext: ext}
}
// special case * when there are no path separators
// filepath.Match never allows * to match a path separator, which is correct
@ -82,16 +88,39 @@ func NewPattern(rawpattern string) Pattern {
rawPattern: cleanpattern,
wildcardRE: regexp.MustCompile(regpattern),
}
// Also support ** with path separators
} else if hasPathSep && strings.Contains(cleanpattern, "**") {
}
// Also support ** with path separators
if hasPathSep && strings.Contains(cleanpattern, "**") {
pattern := regexp.QuoteMeta(cleanpattern)
regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*\\*", ".*", -1))
return &doubleWildcardPattern{
rawPattern: cleanpattern,
wildcardRE: regexp.MustCompile(regpattern),
}
} else {
return &basicPattern{rawPattern: cleanpattern}
}
if hasPathSep && strings.HasPrefix(cleanpattern, sep) {
rel := cleanpattern[1:len(cleanpattern)]
prefix := rel
if strings.HasSuffix(rel, sep) {
rel = rel[0 : len(rel)-1]
} else {
prefix += sep
}
return &pathPrefixPattern{
rawPattern: cleanpattern,
relative: rel,
prefix: prefix,
}
}
return &pathPattern{
rawPattern: cleanpattern,
prefix: cleanpattern + sep,
suffix: sep + cleanpattern,
inner: sep + cleanpattern + sep,
}
}
@ -103,16 +132,45 @@ func convertToPatterns(rawpatterns []string) []Pattern {
return patterns
}
type basicPattern struct {
type pathPrefixPattern struct {
rawPattern string
relative string
prefix string
}
// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *basicPattern) Match(name string) bool {
func (p *pathPrefixPattern) Match(name string) bool {
if name == p.relative || strings.HasPrefix(name, p.prefix) {
return true
}
matched, _ := filepath.Match(p.rawPattern, name)
// Also support matching a parent directory without a wildcard
return matched || strings.HasPrefix(name, p.rawPattern+string(filepath.Separator))
return matched
}
type pathPattern struct {
rawPattern string
prefix string
suffix string
inner string
}
// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathPattern) Match(name string) bool {
if strings.HasPrefix(name, p.prefix) || strings.HasSuffix(name, p.suffix) || strings.Contains(name, p.inner) {
return true
}
matched, _ := filepath.Match(p.rawPattern, name)
return matched
}
type simpleExtPattern struct {
ext string
}
func (p *simpleExtPattern) Match(name string) bool {
return strings.HasSuffix(name, p.ext)
}
type pathlessWildcardPattern struct {

99
filepathfilter/filepathfilter_test.go
View File

@ -10,39 +10,94 @@ import (
)
func TestPatternMatch(t *testing.T) {
assert.True(t, patternMatch("filename.txt", "filename.txt"))
assert.True(t, patternMatch("*.txt", "filename.txt"))
assert.False(t, patternMatch("*.tx", "filename.txt"))
assert.True(t, patternMatch("f*.txt", "filename.txt"))
assert.False(t, patternMatch("g*.txt", "filename.txt"))
assert.True(t, patternMatch("file*", "filename.txt"))
assert.False(t, patternMatch("file", "filename.txt"))
assertPatternMatch(t, "filename.txt", "filename.txt")
assertPatternMatch(t, "*.txt", "filename.txt")
refutePatternMatch(t, "*.tx", "filename.txt")
assertPatternMatch(t, "f*.txt", "filename.txt")
refutePatternMatch(t, "g*.txt", "filename.txt")
assertPatternMatch(t, "file*", "filename.txt")
refutePatternMatch(t, "file", "filename.txt")
// With no path separators, should match in subfolders
assert.True(t, patternMatch("*.txt", "sub/filename.txt"))
assert.False(t, patternMatch("*.tx", "sub/filename.txt"))
assert.True(t, patternMatch("f*.txt", "sub/filename.txt"))
assert.False(t, patternMatch("g*.txt", "sub/filename.txt"))
assert.True(t, patternMatch("file*", "sub/filename.txt"))
assert.False(t, patternMatch("file", "sub/filename.txt"))
assertPatternMatch(t, "*.txt", "sub/filename.txt")
refutePatternMatch(t, "*.tx", "sub/filename.txt")
assertPatternMatch(t, "f*.txt", "sub/filename.txt")
refutePatternMatch(t, "g*.txt", "sub/filename.txt")
assertPatternMatch(t, "file*", "sub/filename.txt")
refutePatternMatch(t, "file", "sub/filename.txt")
// matches only in subdir
assertPatternMatch(t, "sub/*.txt", "sub/filename.txt")
refutePatternMatch(t, "sub/*.txt", "top/sub/filename.txt")
refutePatternMatch(t, "sub/*.txt", "sub/filename.dat")
refutePatternMatch(t, "sub/*.txt", "other/filename.txt")
// Needs wildcard for exact filename
assert.True(t, patternMatch("**/filename.txt", "sub/sub/sub/filename.txt"))
assertPatternMatch(t, "**/filename.txt", "sub/sub/sub/filename.txt")
// Should not match dots to subparts
assert.False(t, patternMatch("*.ign", "sub/shouldignoreme.txt"))
refutePatternMatch(t, "*.ign", "sub/shouldignoreme.txt")
// Path specific
assert.True(t, patternMatch("sub", "sub/filename.txt"))
assert.False(t, patternMatch("sub", "subfilename.txt"))
assertPatternMatch(t, "sub", "sub/")
assertPatternMatch(t, "sub", "sub")
assertPatternMatch(t, "sub", "sub/filename.txt")
assertPatternMatch(t, "sub/", "sub/filename.txt")
assertPatternMatch(t, "sub", "top/sub/filename.txt")
assertPatternMatch(t, "sub/", "top/sub/filename.txt")
assertPatternMatch(t, "sub", "top/sub/")
assertPatternMatch(t, "sub", "top/sub")
assertPatternMatch(t, "/sub", "sub/")
assertPatternMatch(t, "/sub", "sub")
assertPatternMatch(t, "/sub", "sub/filename.txt")
assertPatternMatch(t, "/sub/", "sub/filename.txt")
refutePatternMatch(t, "/sub", "top/sub/filename.txt")
refutePatternMatch(t, "/sub/", "top/sub/filename.txt")
refutePatternMatch(t, "/sub", "top/sub/")
refutePatternMatch(t, "/sub", "top/sub")
refutePatternMatch(t, "sub", "subfilename.txt")
refutePatternMatch(t, "sub/", "subfilename.txt")
refutePatternMatch(t, "/sub", "subfilename.txt")
refutePatternMatch(t, "/sub/", "subfilename.txt")
// nested path
assertPatternMatch(t, "top/sub", "top/sub/filename.txt")
assertPatternMatch(t, "top/sub/", "top/sub/filename.txt")
assertPatternMatch(t, "top/sub", "top/sub/")
assertPatternMatch(t, "top/sub", "top/sub")
assertPatternMatch(t, "top/sub", "root/top/sub/filename.txt")
assertPatternMatch(t, "top/sub/", "root/top/sub/filename.txt")
assertPatternMatch(t, "top/sub", "root/top/sub/")
assertPatternMatch(t, "top/sub", "root/top/sub")
assertPatternMatch(t, "/top/sub", "top/sub/filename.txt")
assertPatternMatch(t, "/top/sub/", "top/sub/filename.txt")
assertPatternMatch(t, "/top/sub", "top/sub/")
assertPatternMatch(t, "/top/sub", "top/sub")
refutePatternMatch(t, "/top/sub", "root/top/sub/filename.txt")
refutePatternMatch(t, "/top/sub/", "root/top/sub/filename.txt")
refutePatternMatch(t, "/top/sub", "root/top/sub/")
refutePatternMatch(t, "/top/sub", "root/top/sub")
refutePatternMatch(t, "top/sub", "top/subfilename.txt")
refutePatternMatch(t, "top/sub/", "top/subfilename.txt")
refutePatternMatch(t, "/top/sub", "top/subfilename.txt")
refutePatternMatch(t, "/top/sub/", "top/subfilename.txt")
// Absolute
assert.True(t, patternMatch("*.dat", "/path/to/sub/.git/test.dat"))
assert.True(t, patternMatch("**/.git", "/path/to/sub/.git"))
assertPatternMatch(t, "*.dat", "/path/to/sub/.git/test.dat")
assertPatternMatch(t, "**/.git", "/path/to/sub/.git")
// Match anything
assert.True(t, patternMatch(".", "path.txt"))
assert.True(t, patternMatch("./", "path.txt"))
assert.True(t, patternMatch(".\\", "path.txt"))
assertPatternMatch(t, ".", "path.txt")
assertPatternMatch(t, "./", "path.txt")
assertPatternMatch(t, ".\\", "path.txt")
}
func assertPatternMatch(t *testing.T, pattern, filename string) {
assert.True(t, patternMatch(pattern, filename), "%q should match pattern %q", filename, pattern)
}
func refutePatternMatch(t *testing.T, pattern, filename string) {
assert.False(t, patternMatch(pattern, filename), "%q should not match pattern %q", filename, pattern)
}
func patternMatch(pattern, filename string) bool {

47
lfsapi/ssh.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
@ -105,13 +106,11 @@ func (c *sshAuthClient) Resolve(e Endpoint, method string) (sshAuthResponse, err
}
func sshGetLFSExeAndArgs(osEnv Env, e Endpoint, method string) (string, []string) {
operation := endpointOperation(e, method)
tracerx.Printf("ssh: %s git-lfs-authenticate %s %s",
e.SshUserAndHost, e.SshPath, operation)
exe, args := sshGetExeAndArgs(osEnv, e)
return exe, append(args,
fmt.Sprintf("git-lfs-authenticate %s %s", e.SshPath, operation))
operation := endpointOperation(e, method)
args = append(args, fmt.Sprintf("git-lfs-authenticate %s %s", e.SshPath, operation))
tracerx.Printf("run_command: %s %s", exe, strings.Join(args, " "))
return exe, args
}
// Return the executable name for ssh on this machine and the base args
@ -129,9 +128,12 @@ func sshGetExeAndArgs(osEnv Env, e Endpoint) (exe string, baseargs []string) {
}
if ssh == "" {
ssh = "ssh"
} else {
basessh := filepath.Base(ssh)
ssh = defaultSSHCmd
}
basessh := filepath.Base(ssh)
if basessh != defaultSSHCmd {
// Strip extension for easier comparison
if ext := filepath.Ext(basessh); len(ext) > 0 {
basessh = basessh[:len(basessh)-len(ext)]
@ -140,7 +142,7 @@ func sshGetExeAndArgs(osEnv Env, e Endpoint) (exe string, baseargs []string) {
isTortoise = strings.EqualFold(basessh, "tortoiseplink")
}
args := make([]string, 0, 4+len(cmdArgs))
args := make([]string, 0, 5+len(cmdArgs))
if len(cmdArgs) > 0 {
args = append(args, cmdArgs...)
}
@ -158,7 +160,30 @@ func sshGetExeAndArgs(osEnv Env, e Endpoint) (exe string, baseargs []string) {
}
args = append(args, e.SshPort)
}
args = append(args, e.SshUserAndHost)
if sep, ok := sshSeparators[basessh]; ok {
// inserts a separator between cli -options and host/cmd commands
// example: $ ssh -p 12345 -- user@host.com git-lfs-authenticate ...
args = append(args, sep, e.SshUserAndHost)
} else {
// no prefix supported, strip leading - off host to prevent cmd like:
// $ git config lfs.url ssh://-proxycmd=whatever
// $ plink -P 12345 -proxycmd=foo git-lfs-authenticate ...
//
// Instead, it'll attempt this, and eventually return an error
// $ plink -P 12345 proxycmd=foo git-lfs-authenticate ...
args = append(args, sshOptPrefixRE.ReplaceAllString(e.SshUserAndHost, ""))
}
return ssh, args
}
const defaultSSHCmd = "ssh"
var (
sshOptPrefixRE = regexp.MustCompile(`\A\-+`)
sshSeparators = map[string]string{
"ssh": "--",
"lfs-ssh-echo": "--", // used in lfs integration tests only
}
)

125
lfsapi/ssh_test.go
View File

@ -2,6 +2,7 @@ package lfsapi
import (
"errors"
"net/url"
"path/filepath"
"testing"
"time"
@ -222,6 +223,7 @@ func TestSSHGetLFSExeAndArgs(t *testing.T) {
exe, args := sshGetLFSExeAndArgs(cli.OSEnv(), endpoint, "GET")
assert.Equal(t, "ssh", exe)
assert.Equal(t, []string{
"--",
"user@foo.com",
"git-lfs-authenticate user/repo download",
}, args)
@ -229,6 +231,7 @@ func TestSSHGetLFSExeAndArgs(t *testing.T) {
exe, args = sshGetLFSExeAndArgs(cli.OSEnv(), endpoint, "HEAD")
assert.Equal(t, "ssh", exe)
assert.Equal(t, []string{
"--",
"user@foo.com",
"git-lfs-authenticate user/repo download",
}, args)
@ -237,6 +240,7 @@ func TestSSHGetLFSExeAndArgs(t *testing.T) {
exe, args = sshGetLFSExeAndArgs(cli.OSEnv(), endpoint, "POST")
assert.Equal(t, "ssh", exe)
assert.Equal(t, []string{
"--",
"user@foo.com",
"git-lfs-authenticate user/repo download",
}, args)
@ -245,6 +249,7 @@ func TestSSHGetLFSExeAndArgs(t *testing.T) {
exe, args = sshGetLFSExeAndArgs(cli.OSEnv(), endpoint, "POST")
assert.Equal(t, "ssh", exe)
assert.Equal(t, []string{
"--",
"user@foo.com",
"git-lfs-authenticate user/repo upload",
}, args)
@ -262,7 +267,7 @@ func TestSSHGetExeAndArgsSsh(t *testing.T) {
exe, args := sshGetExeAndArgs(cli.OSEnv(), endpoint)
assert.Equal(t, "ssh", exe)
assert.Equal(t, []string{"user@foo.com"}, args)
assert.Equal(t, []string{"--", "user@foo.com"}, args)
}
func TestSSHGetExeAndArgsSshCustomPort(t *testing.T) {
@ -278,7 +283,7 @@ func TestSSHGetExeAndArgsSshCustomPort(t *testing.T) {
exe, args := sshGetExeAndArgs(cli.OSEnv(), endpoint)
assert.Equal(t, "ssh", exe)
assert.Equal(t, []string{"-p", "8888", "user@foo.com"}, args)
assert.Equal(t, []string{"-p", "8888", "--", "user@foo.com"}, args)
}
func TestSSHGetExeAndArgsPlink(t *testing.T) {
@ -409,6 +414,122 @@ func TestSSHGetExeAndArgsSshCommandCustomPort(t *testing.T) {
assert.Equal(t, []string{"-p", "8888", "user@foo.com"}, args)
}
func TestSSHGetLFSExeAndArgsWithCustomSSH(t *testing.T) {
cli, err := NewClient(UniqTestEnv(map[string]string{
"GIT_SSH": "not-ssh",
}), nil)
require.Nil(t, err)
u, err := url.Parse("ssh://git@host.com:12345/repo")
require.Nil(t, err)
e := endpointFromSshUrl(u)
t.Logf("ENDPOINT: %+v", e)
assert.Equal(t, "12345", e.SshPort)
assert.Equal(t, "git@host.com", e.SshUserAndHost)
assert.Equal(t, "repo", e.SshPath)
exe, args := sshGetLFSExeAndArgs(cli.OSEnv(), e, "GET")
assert.Equal(t, "not-ssh", exe)
assert.Equal(t, []string{"-p", "12345", "git@host.com", "git-lfs-authenticate repo download"}, args)
}
func TestSSHGetLFSExeAndArgsInvalidOptionsAsHost(t *testing.T) {
cli, err := NewClient(nil, nil)
require.Nil(t, err)
u, err := url.Parse("ssh://-oProxyCommand=gnome-calculator/repo")
require.Nil(t, err)
assert.Equal(t, "-oProxyCommand=gnome-calculator", u.Host)
e := endpointFromSshUrl(u)
t.Logf("ENDPOINT: %+v", e)
assert.Equal(t, "-oProxyCommand=gnome-calculator", e.SshUserAndHost)
assert.Equal(t, "repo", e.SshPath)
exe, args := sshGetLFSExeAndArgs(cli.OSEnv(), e, "GET")
assert.Equal(t, "ssh", exe)
assert.Equal(t, []string{"--", "-oProxyCommand=gnome-calculator", "git-lfs-authenticate repo download"}, args)
}
func TestSSHGetLFSExeAndArgsInvalidOptionsAsHostWithCustomSSH(t *testing.T) {
cli, err := NewClient(UniqTestEnv(map[string]string{
"GIT_SSH": "not-ssh",
}), nil)
require.Nil(t, err)
u, err := url.Parse("ssh://--oProxyCommand=gnome-calculator/repo")
require.Nil(t, err)
assert.Equal(t, "--oProxyCommand=gnome-calculator", u.Host)
e := endpointFromSshUrl(u)
t.Logf("ENDPOINT: %+v", e)
assert.Equal(t, "--oProxyCommand=gnome-calculator", e.SshUserAndHost)
assert.Equal(t, "repo", e.SshPath)
exe, args := sshGetLFSExeAndArgs(cli.OSEnv(), e, "GET")
assert.Equal(t, "not-ssh", exe)
assert.Equal(t, []string{"oProxyCommand=gnome-calculator", "git-lfs-authenticate repo download"}, args)
}
func TestSSHGetExeAndArgsInvalidOptionsAsHost(t *testing.T) {
cli, err := NewClient(nil, nil)
require.Nil(t, err)
u, err := url.Parse("ssh://-oProxyCommand=gnome-calculator")
require.Nil(t, err)
assert.Equal(t, "-oProxyCommand=gnome-calculator", u.Host)
e := endpointFromSshUrl(u)
t.Logf("ENDPOINT: %+v", e)
assert.Equal(t, "-oProxyCommand=gnome-calculator", e.SshUserAndHost)
assert.Equal(t, "", e.SshPath)
exe, args := sshGetExeAndArgs(cli.OSEnv(), e)
assert.Equal(t, "ssh", exe)
assert.Equal(t, []string{"--", "-oProxyCommand=gnome-calculator"}, args)
}
func TestSSHGetExeAndArgsInvalidOptionsAsPath(t *testing.T) {
cli, err := NewClient(nil, nil)
require.Nil(t, err)
u, err := url.Parse("ssh://git@git-host.com/-oProxyCommand=gnome-calculator")
require.Nil(t, err)
assert.Equal(t, "git-host.com", u.Host)
e := endpointFromSshUrl(u)
t.Logf("ENDPOINT: %+v", e)
assert.Equal(t, "git@git-host.com", e.SshUserAndHost)
assert.Equal(t, "-oProxyCommand=gnome-calculator", e.SshPath)
exe, args := sshGetExeAndArgs(cli.OSEnv(), e)
assert.Equal(t, "ssh", exe)
assert.Equal(t, []string{"--", "git@git-host.com"}, args)
}
func TestParseBareSSHUrl(t *testing.T) {
e := endpointFromBareSshUrl("git@git-host.com:repo.git")
t.Logf("endpoint: %+v", e)
assert.Equal(t, "git@git-host.com", e.SshUserAndHost)
assert.Equal(t, "repo.git", e.SshPath)
e = endpointFromBareSshUrl("git@git-host.com/should-be-a-colon.git")
t.Logf("endpoint: %+v", e)
assert.Equal(t, "", e.SshUserAndHost)
assert.Equal(t, "", e.SshPath)
e = endpointFromBareSshUrl("-oProxyCommand=gnome-calculator")
t.Logf("endpoint: %+v", e)
assert.Equal(t, "", e.SshUserAndHost)
assert.Equal(t, "", e.SshPath)
e = endpointFromBareSshUrl("git@git-host.com:-oProxyCommand=gnome-calculator")
t.Logf("endpoint: %+v", e)
assert.Equal(t, "git@git-host.com", e.SshUserAndHost)
assert.Equal(t, "-oProxyCommand=gnome-calculator", e.SshPath)
}
func TestSSHGetExeAndArgsPlinkCommand(t *testing.T) {
plink := filepath.Join("Users", "joebloggs", "bin", "plink.exe")

21
test/cmd/ssh-echo.go → test/cmd/lfs-ssh-echo.go
View File

@ -19,14 +19,29 @@ type sshResponse struct {
func main() {
// expect args:
// ssh-echo -p PORT git@127.0.0.1 git-lfs-authenticate REPO OPERATION
if len(os.Args) != 5 {
// lfs-ssh-echo -p PORT -- git@127.0.0.1 git-lfs-authenticate REPO OPERATION
if len(os.Args) != 6 {
fmt.Fprintf(os.Stderr, "got %d args: %v", len(os.Args), os.Args)
os.Exit(1)
}
if os.Args[1] != "-p" {
fmt.Fprintf(os.Stderr, "$1 expected \"-p\", got %q", os.Args[1])
os.Exit(1)
}
if os.Args[3] != "--" {
fmt.Fprintf(os.Stderr, "$3 expected \"--\", got %q", os.Args[3])
os.Exit(1)
}
if os.Args[4] != "git@127.0.0.1" {
fmt.Fprintf(os.Stderr, "$4 expected \"git@127.0.0.1\", got %q", os.Args[4])
os.Exit(1)
}
// just "git-lfs-authenticate REPO OPERATION"
authLine := strings.Split(os.Args[4], " ")
authLine := strings.Split(os.Args[5], " ")
if len(authLine) < 13 {
fmt.Fprintf(os.Stderr, "bad git-lfs-authenticate line: %s\nargs: %v", authLine, os.Args)
}

9
test/cmd/lfs-ssh-proxy-test.go Normal file
View File

@ -0,0 +1,9 @@
// +build testtools
package main
import "fmt"
func main() {
fmt.Println("SSH PROXY TEST called")
}

2
test/test-env.sh
View File

@ -673,7 +673,7 @@ begin_test "env with multiple ssh remotes"
SSH=git@git-server.com:user/repo.git
Endpoint (other)=https://other-git-server.com/user/repo.git/info/lfs (auth=none)
SSH=git@other-git-server.com:user/repo.git
GIT_SSH=ssh-echo'
GIT_SSH=lfs-ssh-echo'
contains_same_elements "$expected" "$(git lfs env | grep -e "Endpoint" -e "SSH=")"
)

33
test/test-ssh.sh Executable file
View File

@ -0,0 +1,33 @@
#!/usr/bin/env bash
. "test/testlib.sh"
begin_test "ssh with proxy command in lfs.url"
(
set -e
reponame="batch-ssh-proxy"
setup_remote_repo "$reponame"
clone_repo "$reponame" "$reponame"
sshurl="${GITSERVER/http:\/\//ssh://-oProxyCommand=ssh-proxy-test/}/$reponame"
echo $sshurl
git config lfs.url "$sshurl"
contents="test"
oid="$(calc_oid "$contents")"
git lfs track "*.dat"
printf "$contents" > test.dat
git add .gitattributes test.dat
git commit -m "initial commit"
git push origin master 2>&1 | tee push.log
if [ "0" -eq "${PIPESTATUS[0]}" ]; then
echo >&2 "fatal: push succeeded"
exit 1
fi
grep "got 4 args" push.log
grep "lfs-ssh-echo -- -oProxyCommand" push.log
)
end_test

2
test/testenv.sh
View File

@ -120,7 +120,7 @@ TESTHOME="$REMOTEDIR/home"
GIT_CONFIG_NOSYSTEM=1
GIT_TERMINAL_PROMPT=0
GIT_SSH=ssh-echo
GIT_SSH=lfs-ssh-echo
APPVEYOR_REPO_COMMIT_MESSAGE="test: env test should look for GIT_SSH too"
export CREDSDIR

52
tools/filetools.go
View File

@ -139,14 +139,16 @@ type FastWalkCallback func(parentDir string, info os.FileInfo, err error)
// determine absolute path rather than tracking it yourself
// * Automatically ignores any .git directories
// * Respects .gitignore contents and skips ignored files/dirs
func FastWalkGitRepo(dir string, cb FastWalkCallback) {
//
// rootDir - Absolute path to the top of the repository working directory
func FastWalkGitRepo(rootDir string, cb FastWalkCallback) {
// Ignore all git metadata including subrepos
excludePaths := []filepathfilter.Pattern{
filepathfilter.NewPattern(".git"),
filepathfilter.NewPattern(filepath.Join("**", ".git")),
}
fileCh := fastWalkWithExcludeFiles(dir, ".gitignore", excludePaths)
fileCh := fastWalkWithExcludeFiles(rootDir, ".gitignore", excludePaths)
for file := range fileCh {
cb(file.ParentDir, file.Info, file.Err)
}
@ -164,17 +166,20 @@ type fastWalkInfo struct {
// fastWalkWithExcludeFiles walks the contents of a dir, respecting
// include/exclude patterns and also loading new exlude patterns from files
// named excludeFilename in directories walked
func fastWalkWithExcludeFiles(dir, excludeFilename string,
//
// rootDir - Absolute path to the top of the repository working directory
func fastWalkWithExcludeFiles(rootDir, excludeFilename string,
excludePaths []filepathfilter.Pattern) <-chan fastWalkInfo {
fiChan := make(chan fastWalkInfo, 256)
go fastWalkFromRoot(dir, excludeFilename, excludePaths, fiChan)
go fastWalkFromRoot(rootDir, excludeFilename, excludePaths, fiChan)
return fiChan
}
func fastWalkFromRoot(dir string, excludeFilename string,
// rootDir - Absolute path to the top of the repository working directory
func fastWalkFromRoot(rootDir string, excludeFilename string,
excludePaths []filepathfilter.Pattern, fiChan chan<- fastWalkInfo) {
dirFi, err := os.Stat(dir)
dirFi, err := os.Stat(rootDir)
if err != nil {
fiChan <- fastWalkInfo{Err: err}
return
@ -182,7 +187,7 @@ func fastWalkFromRoot(dir string, excludeFilename string,
// This waitgroup will be incremented for each nested goroutine
var waitg sync.WaitGroup
fastWalkFileOrDir(filepath.Dir(dir), dirFi, excludeFilename, excludePaths, fiChan, &waitg)
fastWalkFileOrDir(true, rootDir, "", dirFi, excludeFilename, excludePaths, fiChan, &waitg)
waitg.Wait()
close(fiChan)
}
@ -193,26 +198,42 @@ func fastWalkFromRoot(dir string, excludeFilename string,
// the excludePaths with its content before (parallel) recursing into contents
// Also splits large directories into multiple goroutines.
// Increments waitg.Add(1) for each new goroutine launched internally
func fastWalkFileOrDir(parentDir string, itemFi os.FileInfo, excludeFilename string,
//
// rootDir - Absolute path to the top of the repository working directory
// workDir - Relative path inside the repository
func fastWalkFileOrDir(isRoot bool, rootDir, workDir string, itemFi os.FileInfo, excludeFilename string,
excludePaths []filepathfilter.Pattern, fiChan chan<- fastWalkInfo, waitg *sync.WaitGroup) {
fullPath := filepath.Join(parentDir, itemFi.Name())
var fullPath string // Absolute path to the current file or dir
var parentWorkDir string // Absolute path to the workDir inside the repository
if isRoot {
fullPath = rootDir
} else {
parentWorkDir = filepath.Join(rootDir, workDir)
fullPath = filepath.Join(parentWorkDir, itemFi.Name())
}
if !filepathfilter.NewFromPatterns(nil, excludePaths).Allows(fullPath) {
workPath := filepath.Join(workDir, itemFi.Name())
if !filepathfilter.NewFromPatterns(nil, excludePaths).Allows(workPath) {
return
}
fiChan <- fastWalkInfo{ParentDir: parentDir, Info: itemFi}
fiChan <- fastWalkInfo{ParentDir: parentWorkDir, Info: itemFi}
if !itemFi.IsDir() {
// Nothing more to do if this is not a dir
return
}
var childWorkDir string
if !isRoot {
childWorkDir = filepath.Join(workDir, itemFi.Name())
}
if len(excludeFilename) > 0 {
possibleExcludeFile := filepath.Join(fullPath, excludeFilename)
var err error
excludePaths, err = loadExcludeFilename(possibleExcludeFile, fullPath, excludePaths)
excludePaths, err = loadExcludeFilename(possibleExcludeFile, childWorkDir, excludePaths)
if err != nil {
fiChan <- fastWalkInfo{Err: err}
}
@ -231,12 +252,13 @@ func fastWalkFileOrDir(parentDir string, itemFi os.FileInfo, excludeFilename str
// The number of items in a dir we process in each goroutine
jobSize := 100
for children, err := df.Readdir(jobSize); err == nil; children, err = df.Readdir(jobSize) {
// Parallelise all dirs, and chop large dirs into batches
waitg.Add(1)
go func(subitems []os.FileInfo) {
for _, childFi := range subitems {
fastWalkFileOrDir(fullPath, childFi, excludeFilename, excludePaths, fiChan, waitg)
fastWalkFileOrDir(false, rootDir, childWorkDir, childFi, excludeFilename, excludePaths, fiChan, waitg)
}
waitg.Done()
}(children)
@ -251,7 +273,7 @@ func fastWalkFileOrDir(parentDir string, itemFi os.FileInfo, excludeFilename str
// revised array of exclude paths if there are any changes.
// If any changes are made a copy of the array is taken so the original is not
// modified
func loadExcludeFilename(filename, parentDir string, excludePaths []filepathfilter.Pattern) ([]filepathfilter.Pattern, error) {
func loadExcludeFilename(filename, workDir string, excludePaths []filepathfilter.Pattern) ([]filepathfilter.Pattern, error) {
f, err := os.OpenFile(filename, os.O_RDONLY, 0644)
if err != nil {
if os.IsNotExist(err) {
@ -284,7 +306,7 @@ func loadExcludeFilename(filename, parentDir string, excludePaths []filepathfilt
// Allow for both styles of separator at this point
if strings.ContainsAny(path, "/\\") ||
!strings.Contains(path, "*") {
path = filepath.Join(parentDir, line)
path = filepath.Join(workDir, line)
}
retPaths = append(retPaths, filepathfilter.NewPattern(path))
}

3
tools/filetools_test.go
View File

@ -109,6 +109,9 @@ func TestFastWalkGitRepo(t *testing.T) {
"filethatweignore.ign",
"foldercontainingignored",
"foldercontainingignored/notthisone.ign",
"foldercontainingignored/ignoredfolder",
"foldercontainingignored/ignoredfolder/file1.txt",
"foldercontainingignored/ignoredfolder/file2.txt",
"ignoredfolder",
"ignoredfolder/file1.txt",
"ignoredfolder/file2.txt",