diff --git a/docs/man/git-lfs.1.ronn b/docs/man/git-lfs.1.ronn index b4b21ca0..3ec6f5b5 100644 --- a/docs/man/git-lfs.1.ronn +++ b/docs/man/git-lfs.1.ronn @@ -41,6 +41,10 @@ commands and low level ("plumbing") commands. Check GIT LFS files for consistency. * git-lfs-install(1): Install Git LFS configuration. +* git-lfs-lock(1): + Set a file as "locked" on the Git LFS server. +* git-lfs-locks(1): + List currently locked files from the Git LFS server. * git-lfs-logs(1): Show errors from the git-lfs command. * git-lfs-ls-files(1): @@ -53,6 +57,8 @@ commands and low level ("plumbing") commands. Show the status of Git LFS files in the working tree. * git-lfs-track(1): View or add Git LFS paths to Git attributes. +* git-lfs-unlock(1): + Remove "locked" setting for a file on the Git LFS server. * git-lfs-untrack(1): Remove Git LFS paths from Git Attributes. * git-lfs-update(1): diff --git a/filepathfilter/bench_test.go b/filepathfilter/bench_test.go index 83899ecc..567bbb55 100644 --- a/filepathfilter/bench_test.go +++ b/filepathfilter/bench_test.go @@ -10,7 +10,27 @@ import ( "github.com/git-lfs/git-lfs/tools" ) -func BenchmarkFilterIncludeWildcardOnly(b *testing.B) { +func BenchmarkFilterSimplePath(b *testing.B) { + files := benchmarkTree(b) + filter := filepathfilter.New([]string{"lfs"}, nil) + for i := 0; i < b.N; i++ { + for _, f := range files { + filter.Allows(f) + } + } +} + +func BenchmarkPatternSimplePath(b *testing.B) { + files := benchmarkTree(b) + pattern := filepathfilter.NewPattern("lfs") + for i := 0; i < b.N; i++ { + for _, f := range files { + pattern.Match(f) + } + } +} + +func BenchmarkFilterSimpleExtension(b *testing.B) { files := benchmarkTree(b) filter := filepathfilter.New([]string{"*.go"}, nil) for i := 0; i < b.N; i++ { @@ -20,7 +40,37 @@ func BenchmarkFilterIncludeWildcardOnly(b *testing.B) { } } -func BenchmarkFilterIncludeDoubleAsterisk(b *testing.B) { +func BenchmarkPatternSimpleExtension(b *testing.B) { + files := benchmarkTree(b) + pattern := filepathfilter.NewPattern("*.go") + for i := 0; i < b.N; i++ { + for _, f := range files { + pattern.Match(f) + } + } +} + +func BenchmarkFilterComplexExtension(b *testing.B) { + files := benchmarkTree(b) + filter := filepathfilter.New([]string{"*.travis.yml"}, nil) + for i := 0; i < b.N; i++ { + for _, f := range files { + filter.Allows(f) + } + } +} + +func BenchmarkPatternComplexExtension(b *testing.B) { + files := benchmarkTree(b) + pattern := filepathfilter.NewPattern("*.travis.yml") + for i := 0; i < b.N; i++ { + for _, f := range files { + pattern.Match(f) + } + } +} + +func BenchmarkFilterDoubleAsterisk(b *testing.B) { files := benchmarkTree(b) filter := filepathfilter.New([]string{"**/README.md"}, nil) for i := 0; i < b.N; i++ { @@ -30,6 +80,16 @@ func BenchmarkFilterIncludeDoubleAsterisk(b *testing.B) { } } +func BenchmarkPatternDoubleAsterisk(b *testing.B) { + files := benchmarkTree(b) + pattern := filepathfilter.NewPattern("**/README.md") + for i := 0; i < b.N; i++ { + for _, f := range files { + pattern.Match(f) + } + } +} + var ( benchmarkFiles []string benchmarkMu sync.Mutex diff --git a/filepathfilter/filepathfilter.go b/filepathfilter/filepathfilter.go index aee2616f..b9ec5325 100644 --- a/filepathfilter/filepathfilter.go +++ b/filepathfilter/filepathfilter.go @@ -67,7 +67,13 @@ func NewPattern(rawpattern string) Pattern { return noOpMatcher{} } - hasPathSep := strings.Contains(cleanpattern, string(filepath.Separator)) + sep := string(filepath.Separator) + hasPathSep := strings.Contains(cleanpattern, sep) + ext := filepath.Ext(cleanpattern) + plen := len(cleanpattern) + if plen > 1 && !hasPathSep && strings.HasPrefix(cleanpattern, "*") && cleanpattern[1:plen] == ext { + return &simpleExtPattern{ext: ext} + } // special case * when there are no path separators // filepath.Match never allows * to match a path separator, which is correct @@ -82,16 +88,39 @@ func NewPattern(rawpattern string) Pattern { rawPattern: cleanpattern, wildcardRE: regexp.MustCompile(regpattern), } - // Also support ** with path separators - } else if hasPathSep && strings.Contains(cleanpattern, "**") { + } + + // Also support ** with path separators + if hasPathSep && strings.Contains(cleanpattern, "**") { pattern := regexp.QuoteMeta(cleanpattern) regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*\\*", ".*", -1)) return &doubleWildcardPattern{ rawPattern: cleanpattern, wildcardRE: regexp.MustCompile(regpattern), } - } else { - return &basicPattern{rawPattern: cleanpattern} + } + + if hasPathSep && strings.HasPrefix(cleanpattern, sep) { + rel := cleanpattern[1:len(cleanpattern)] + prefix := rel + if strings.HasSuffix(rel, sep) { + rel = rel[0 : len(rel)-1] + } else { + prefix += sep + } + + return &pathPrefixPattern{ + rawPattern: cleanpattern, + relative: rel, + prefix: prefix, + } + } + + return &pathPattern{ + rawPattern: cleanpattern, + prefix: cleanpattern + sep, + suffix: sep + cleanpattern, + inner: sep + cleanpattern + sep, } } @@ -103,16 +132,45 @@ func convertToPatterns(rawpatterns []string) []Pattern { return patterns } -type basicPattern struct { +type pathPrefixPattern struct { rawPattern string + relative string + prefix string } // Match is a revised version of filepath.Match which makes it behave more // like gitignore -func (p *basicPattern) Match(name string) bool { +func (p *pathPrefixPattern) Match(name string) bool { + if name == p.relative || strings.HasPrefix(name, p.prefix) { + return true + } matched, _ := filepath.Match(p.rawPattern, name) - // Also support matching a parent directory without a wildcard - return matched || strings.HasPrefix(name, p.rawPattern+string(filepath.Separator)) + return matched +} + +type pathPattern struct { + rawPattern string + prefix string + suffix string + inner string +} + +// Match is a revised version of filepath.Match which makes it behave more +// like gitignore +func (p *pathPattern) Match(name string) bool { + if strings.HasPrefix(name, p.prefix) || strings.HasSuffix(name, p.suffix) || strings.Contains(name, p.inner) { + return true + } + matched, _ := filepath.Match(p.rawPattern, name) + return matched +} + +type simpleExtPattern struct { + ext string +} + +func (p *simpleExtPattern) Match(name string) bool { + return strings.HasSuffix(name, p.ext) } type pathlessWildcardPattern struct { diff --git a/filepathfilter/filepathfilter_test.go b/filepathfilter/filepathfilter_test.go index ab678cfc..7ab3df0f 100644 --- a/filepathfilter/filepathfilter_test.go +++ b/filepathfilter/filepathfilter_test.go @@ -10,39 +10,94 @@ import ( ) func TestPatternMatch(t *testing.T) { - assert.True(t, patternMatch("filename.txt", "filename.txt")) - assert.True(t, patternMatch("*.txt", "filename.txt")) - assert.False(t, patternMatch("*.tx", "filename.txt")) - assert.True(t, patternMatch("f*.txt", "filename.txt")) - assert.False(t, patternMatch("g*.txt", "filename.txt")) - assert.True(t, patternMatch("file*", "filename.txt")) - assert.False(t, patternMatch("file", "filename.txt")) + assertPatternMatch(t, "filename.txt", "filename.txt") + assertPatternMatch(t, "*.txt", "filename.txt") + refutePatternMatch(t, "*.tx", "filename.txt") + assertPatternMatch(t, "f*.txt", "filename.txt") + refutePatternMatch(t, "g*.txt", "filename.txt") + assertPatternMatch(t, "file*", "filename.txt") + refutePatternMatch(t, "file", "filename.txt") // With no path separators, should match in subfolders - assert.True(t, patternMatch("*.txt", "sub/filename.txt")) - assert.False(t, patternMatch("*.tx", "sub/filename.txt")) - assert.True(t, patternMatch("f*.txt", "sub/filename.txt")) - assert.False(t, patternMatch("g*.txt", "sub/filename.txt")) - assert.True(t, patternMatch("file*", "sub/filename.txt")) - assert.False(t, patternMatch("file", "sub/filename.txt")) + assertPatternMatch(t, "*.txt", "sub/filename.txt") + refutePatternMatch(t, "*.tx", "sub/filename.txt") + assertPatternMatch(t, "f*.txt", "sub/filename.txt") + refutePatternMatch(t, "g*.txt", "sub/filename.txt") + assertPatternMatch(t, "file*", "sub/filename.txt") + refutePatternMatch(t, "file", "sub/filename.txt") + + // matches only in subdir + assertPatternMatch(t, "sub/*.txt", "sub/filename.txt") + refutePatternMatch(t, "sub/*.txt", "top/sub/filename.txt") + refutePatternMatch(t, "sub/*.txt", "sub/filename.dat") + refutePatternMatch(t, "sub/*.txt", "other/filename.txt") + // Needs wildcard for exact filename - assert.True(t, patternMatch("**/filename.txt", "sub/sub/sub/filename.txt")) + assertPatternMatch(t, "**/filename.txt", "sub/sub/sub/filename.txt") // Should not match dots to subparts - assert.False(t, patternMatch("*.ign", "sub/shouldignoreme.txt")) + refutePatternMatch(t, "*.ign", "sub/shouldignoreme.txt") // Path specific - assert.True(t, patternMatch("sub", "sub/filename.txt")) - assert.False(t, patternMatch("sub", "subfilename.txt")) + assertPatternMatch(t, "sub", "sub/") + assertPatternMatch(t, "sub", "sub") + assertPatternMatch(t, "sub", "sub/filename.txt") + assertPatternMatch(t, "sub/", "sub/filename.txt") + assertPatternMatch(t, "sub", "top/sub/filename.txt") + assertPatternMatch(t, "sub/", "top/sub/filename.txt") + assertPatternMatch(t, "sub", "top/sub/") + assertPatternMatch(t, "sub", "top/sub") + assertPatternMatch(t, "/sub", "sub/") + assertPatternMatch(t, "/sub", "sub") + assertPatternMatch(t, "/sub", "sub/filename.txt") + assertPatternMatch(t, "/sub/", "sub/filename.txt") + refutePatternMatch(t, "/sub", "top/sub/filename.txt") + refutePatternMatch(t, "/sub/", "top/sub/filename.txt") + refutePatternMatch(t, "/sub", "top/sub/") + refutePatternMatch(t, "/sub", "top/sub") + refutePatternMatch(t, "sub", "subfilename.txt") + refutePatternMatch(t, "sub/", "subfilename.txt") + refutePatternMatch(t, "/sub", "subfilename.txt") + refutePatternMatch(t, "/sub/", "subfilename.txt") + + // nested path + assertPatternMatch(t, "top/sub", "top/sub/filename.txt") + assertPatternMatch(t, "top/sub/", "top/sub/filename.txt") + assertPatternMatch(t, "top/sub", "top/sub/") + assertPatternMatch(t, "top/sub", "top/sub") + assertPatternMatch(t, "top/sub", "root/top/sub/filename.txt") + assertPatternMatch(t, "top/sub/", "root/top/sub/filename.txt") + assertPatternMatch(t, "top/sub", "root/top/sub/") + assertPatternMatch(t, "top/sub", "root/top/sub") + assertPatternMatch(t, "/top/sub", "top/sub/filename.txt") + assertPatternMatch(t, "/top/sub/", "top/sub/filename.txt") + assertPatternMatch(t, "/top/sub", "top/sub/") + assertPatternMatch(t, "/top/sub", "top/sub") + refutePatternMatch(t, "/top/sub", "root/top/sub/filename.txt") + refutePatternMatch(t, "/top/sub/", "root/top/sub/filename.txt") + refutePatternMatch(t, "/top/sub", "root/top/sub/") + refutePatternMatch(t, "/top/sub", "root/top/sub") + refutePatternMatch(t, "top/sub", "top/subfilename.txt") + refutePatternMatch(t, "top/sub/", "top/subfilename.txt") + refutePatternMatch(t, "/top/sub", "top/subfilename.txt") + refutePatternMatch(t, "/top/sub/", "top/subfilename.txt") // Absolute - assert.True(t, patternMatch("*.dat", "/path/to/sub/.git/test.dat")) - assert.True(t, patternMatch("**/.git", "/path/to/sub/.git")) + assertPatternMatch(t, "*.dat", "/path/to/sub/.git/test.dat") + assertPatternMatch(t, "**/.git", "/path/to/sub/.git") // Match anything - assert.True(t, patternMatch(".", "path.txt")) - assert.True(t, patternMatch("./", "path.txt")) - assert.True(t, patternMatch(".\\", "path.txt")) + assertPatternMatch(t, ".", "path.txt") + assertPatternMatch(t, "./", "path.txt") + assertPatternMatch(t, ".\\", "path.txt") +} + +func assertPatternMatch(t *testing.T, pattern, filename string) { + assert.True(t, patternMatch(pattern, filename), "%q should match pattern %q", filename, pattern) +} + +func refutePatternMatch(t *testing.T, pattern, filename string) { + assert.False(t, patternMatch(pattern, filename), "%q should not match pattern %q", filename, pattern) } func patternMatch(pattern, filename string) bool { diff --git a/lfsapi/ssh.go b/lfsapi/ssh.go index 975ad7ba..f37a4cac 100644 --- a/lfsapi/ssh.go +++ b/lfsapi/ssh.go @@ -6,6 +6,7 @@ import ( "fmt" "os/exec" "path/filepath" + "regexp" "strings" "time" @@ -105,13 +106,11 @@ func (c *sshAuthClient) Resolve(e Endpoint, method string) (sshAuthResponse, err } func sshGetLFSExeAndArgs(osEnv Env, e Endpoint, method string) (string, []string) { - operation := endpointOperation(e, method) - tracerx.Printf("ssh: %s git-lfs-authenticate %s %s", - e.SshUserAndHost, e.SshPath, operation) - exe, args := sshGetExeAndArgs(osEnv, e) - return exe, append(args, - fmt.Sprintf("git-lfs-authenticate %s %s", e.SshPath, operation)) + operation := endpointOperation(e, method) + args = append(args, fmt.Sprintf("git-lfs-authenticate %s %s", e.SshPath, operation)) + tracerx.Printf("run_command: %s %s", exe, strings.Join(args, " ")) + return exe, args } // Return the executable name for ssh on this machine and the base args @@ -129,9 +128,12 @@ func sshGetExeAndArgs(osEnv Env, e Endpoint) (exe string, baseargs []string) { } if ssh == "" { - ssh = "ssh" - } else { - basessh := filepath.Base(ssh) + ssh = defaultSSHCmd + } + + basessh := filepath.Base(ssh) + + if basessh != defaultSSHCmd { // Strip extension for easier comparison if ext := filepath.Ext(basessh); len(ext) > 0 { basessh = basessh[:len(basessh)-len(ext)] @@ -140,7 +142,7 @@ func sshGetExeAndArgs(osEnv Env, e Endpoint) (exe string, baseargs []string) { isTortoise = strings.EqualFold(basessh, "tortoiseplink") } - args := make([]string, 0, 4+len(cmdArgs)) + args := make([]string, 0, 5+len(cmdArgs)) if len(cmdArgs) > 0 { args = append(args, cmdArgs...) } @@ -158,7 +160,30 @@ func sshGetExeAndArgs(osEnv Env, e Endpoint) (exe string, baseargs []string) { } args = append(args, e.SshPort) } - args = append(args, e.SshUserAndHost) + + if sep, ok := sshSeparators[basessh]; ok { + // inserts a separator between cli -options and host/cmd commands + // example: $ ssh -p 12345 -- user@host.com git-lfs-authenticate ... + args = append(args, sep, e.SshUserAndHost) + } else { + // no prefix supported, strip leading - off host to prevent cmd like: + // $ git config lfs.url ssh://-proxycmd=whatever + // $ plink -P 12345 -proxycmd=foo git-lfs-authenticate ... + // + // Instead, it'll attempt this, and eventually return an error + // $ plink -P 12345 proxycmd=foo git-lfs-authenticate ... + args = append(args, sshOptPrefixRE.ReplaceAllString(e.SshUserAndHost, "")) + } return ssh, args } + +const defaultSSHCmd = "ssh" + +var ( + sshOptPrefixRE = regexp.MustCompile(`\A\-+`) + sshSeparators = map[string]string{ + "ssh": "--", + "lfs-ssh-echo": "--", // used in lfs integration tests only + } +) diff --git a/lfsapi/ssh_test.go b/lfsapi/ssh_test.go index 5b90efe8..85155d03 100644 --- a/lfsapi/ssh_test.go +++ b/lfsapi/ssh_test.go @@ -2,6 +2,7 @@ package lfsapi import ( "errors" + "net/url" "path/filepath" "testing" "time" @@ -222,6 +223,7 @@ func TestSSHGetLFSExeAndArgs(t *testing.T) { exe, args := sshGetLFSExeAndArgs(cli.OSEnv(), endpoint, "GET") assert.Equal(t, "ssh", exe) assert.Equal(t, []string{ + "--", "user@foo.com", "git-lfs-authenticate user/repo download", }, args) @@ -229,6 +231,7 @@ func TestSSHGetLFSExeAndArgs(t *testing.T) { exe, args = sshGetLFSExeAndArgs(cli.OSEnv(), endpoint, "HEAD") assert.Equal(t, "ssh", exe) assert.Equal(t, []string{ + "--", "user@foo.com", "git-lfs-authenticate user/repo download", }, args) @@ -237,6 +240,7 @@ func TestSSHGetLFSExeAndArgs(t *testing.T) { exe, args = sshGetLFSExeAndArgs(cli.OSEnv(), endpoint, "POST") assert.Equal(t, "ssh", exe) assert.Equal(t, []string{ + "--", "user@foo.com", "git-lfs-authenticate user/repo download", }, args) @@ -245,6 +249,7 @@ func TestSSHGetLFSExeAndArgs(t *testing.T) { exe, args = sshGetLFSExeAndArgs(cli.OSEnv(), endpoint, "POST") assert.Equal(t, "ssh", exe) assert.Equal(t, []string{ + "--", "user@foo.com", "git-lfs-authenticate user/repo upload", }, args) @@ -262,7 +267,7 @@ func TestSSHGetExeAndArgsSsh(t *testing.T) { exe, args := sshGetExeAndArgs(cli.OSEnv(), endpoint) assert.Equal(t, "ssh", exe) - assert.Equal(t, []string{"user@foo.com"}, args) + assert.Equal(t, []string{"--", "user@foo.com"}, args) } func TestSSHGetExeAndArgsSshCustomPort(t *testing.T) { @@ -278,7 +283,7 @@ func TestSSHGetExeAndArgsSshCustomPort(t *testing.T) { exe, args := sshGetExeAndArgs(cli.OSEnv(), endpoint) assert.Equal(t, "ssh", exe) - assert.Equal(t, []string{"-p", "8888", "user@foo.com"}, args) + assert.Equal(t, []string{"-p", "8888", "--", "user@foo.com"}, args) } func TestSSHGetExeAndArgsPlink(t *testing.T) { @@ -409,6 +414,122 @@ func TestSSHGetExeAndArgsSshCommandCustomPort(t *testing.T) { assert.Equal(t, []string{"-p", "8888", "user@foo.com"}, args) } +func TestSSHGetLFSExeAndArgsWithCustomSSH(t *testing.T) { + cli, err := NewClient(UniqTestEnv(map[string]string{ + "GIT_SSH": "not-ssh", + }), nil) + require.Nil(t, err) + + u, err := url.Parse("ssh://git@host.com:12345/repo") + require.Nil(t, err) + + e := endpointFromSshUrl(u) + t.Logf("ENDPOINT: %+v", e) + assert.Equal(t, "12345", e.SshPort) + assert.Equal(t, "git@host.com", e.SshUserAndHost) + assert.Equal(t, "repo", e.SshPath) + + exe, args := sshGetLFSExeAndArgs(cli.OSEnv(), e, "GET") + assert.Equal(t, "not-ssh", exe) + assert.Equal(t, []string{"-p", "12345", "git@host.com", "git-lfs-authenticate repo download"}, args) +} + +func TestSSHGetLFSExeAndArgsInvalidOptionsAsHost(t *testing.T) { + cli, err := NewClient(nil, nil) + require.Nil(t, err) + + u, err := url.Parse("ssh://-oProxyCommand=gnome-calculator/repo") + require.Nil(t, err) + assert.Equal(t, "-oProxyCommand=gnome-calculator", u.Host) + + e := endpointFromSshUrl(u) + t.Logf("ENDPOINT: %+v", e) + assert.Equal(t, "-oProxyCommand=gnome-calculator", e.SshUserAndHost) + assert.Equal(t, "repo", e.SshPath) + + exe, args := sshGetLFSExeAndArgs(cli.OSEnv(), e, "GET") + assert.Equal(t, "ssh", exe) + assert.Equal(t, []string{"--", "-oProxyCommand=gnome-calculator", "git-lfs-authenticate repo download"}, args) +} + +func TestSSHGetLFSExeAndArgsInvalidOptionsAsHostWithCustomSSH(t *testing.T) { + cli, err := NewClient(UniqTestEnv(map[string]string{ + "GIT_SSH": "not-ssh", + }), nil) + require.Nil(t, err) + + u, err := url.Parse("ssh://--oProxyCommand=gnome-calculator/repo") + require.Nil(t, err) + assert.Equal(t, "--oProxyCommand=gnome-calculator", u.Host) + + e := endpointFromSshUrl(u) + t.Logf("ENDPOINT: %+v", e) + assert.Equal(t, "--oProxyCommand=gnome-calculator", e.SshUserAndHost) + assert.Equal(t, "repo", e.SshPath) + + exe, args := sshGetLFSExeAndArgs(cli.OSEnv(), e, "GET") + assert.Equal(t, "not-ssh", exe) + assert.Equal(t, []string{"oProxyCommand=gnome-calculator", "git-lfs-authenticate repo download"}, args) +} + +func TestSSHGetExeAndArgsInvalidOptionsAsHost(t *testing.T) { + cli, err := NewClient(nil, nil) + require.Nil(t, err) + + u, err := url.Parse("ssh://-oProxyCommand=gnome-calculator") + require.Nil(t, err) + assert.Equal(t, "-oProxyCommand=gnome-calculator", u.Host) + + e := endpointFromSshUrl(u) + t.Logf("ENDPOINT: %+v", e) + assert.Equal(t, "-oProxyCommand=gnome-calculator", e.SshUserAndHost) + assert.Equal(t, "", e.SshPath) + + exe, args := sshGetExeAndArgs(cli.OSEnv(), e) + assert.Equal(t, "ssh", exe) + assert.Equal(t, []string{"--", "-oProxyCommand=gnome-calculator"}, args) +} + +func TestSSHGetExeAndArgsInvalidOptionsAsPath(t *testing.T) { + cli, err := NewClient(nil, nil) + require.Nil(t, err) + + u, err := url.Parse("ssh://git@git-host.com/-oProxyCommand=gnome-calculator") + require.Nil(t, err) + assert.Equal(t, "git-host.com", u.Host) + + e := endpointFromSshUrl(u) + t.Logf("ENDPOINT: %+v", e) + assert.Equal(t, "git@git-host.com", e.SshUserAndHost) + assert.Equal(t, "-oProxyCommand=gnome-calculator", e.SshPath) + + exe, args := sshGetExeAndArgs(cli.OSEnv(), e) + assert.Equal(t, "ssh", exe) + assert.Equal(t, []string{"--", "git@git-host.com"}, args) +} + +func TestParseBareSSHUrl(t *testing.T) { + e := endpointFromBareSshUrl("git@git-host.com:repo.git") + t.Logf("endpoint: %+v", e) + assert.Equal(t, "git@git-host.com", e.SshUserAndHost) + assert.Equal(t, "repo.git", e.SshPath) + + e = endpointFromBareSshUrl("git@git-host.com/should-be-a-colon.git") + t.Logf("endpoint: %+v", e) + assert.Equal(t, "", e.SshUserAndHost) + assert.Equal(t, "", e.SshPath) + + e = endpointFromBareSshUrl("-oProxyCommand=gnome-calculator") + t.Logf("endpoint: %+v", e) + assert.Equal(t, "", e.SshUserAndHost) + assert.Equal(t, "", e.SshPath) + + e = endpointFromBareSshUrl("git@git-host.com:-oProxyCommand=gnome-calculator") + t.Logf("endpoint: %+v", e) + assert.Equal(t, "git@git-host.com", e.SshUserAndHost) + assert.Equal(t, "-oProxyCommand=gnome-calculator", e.SshPath) +} + func TestSSHGetExeAndArgsPlinkCommand(t *testing.T) { plink := filepath.Join("Users", "joebloggs", "bin", "plink.exe") diff --git a/test/cmd/ssh-echo.go b/test/cmd/lfs-ssh-echo.go similarity index 67% rename from test/cmd/ssh-echo.go rename to test/cmd/lfs-ssh-echo.go index d76afa47..292d5961 100644 --- a/test/cmd/ssh-echo.go +++ b/test/cmd/lfs-ssh-echo.go @@ -19,14 +19,29 @@ type sshResponse struct { func main() { // expect args: - // ssh-echo -p PORT git@127.0.0.1 git-lfs-authenticate REPO OPERATION - if len(os.Args) != 5 { + // lfs-ssh-echo -p PORT -- git@127.0.0.1 git-lfs-authenticate REPO OPERATION + if len(os.Args) != 6 { fmt.Fprintf(os.Stderr, "got %d args: %v", len(os.Args), os.Args) os.Exit(1) } + if os.Args[1] != "-p" { + fmt.Fprintf(os.Stderr, "$1 expected \"-p\", got %q", os.Args[1]) + os.Exit(1) + } + + if os.Args[3] != "--" { + fmt.Fprintf(os.Stderr, "$3 expected \"--\", got %q", os.Args[3]) + os.Exit(1) + } + + if os.Args[4] != "git@127.0.0.1" { + fmt.Fprintf(os.Stderr, "$4 expected \"git@127.0.0.1\", got %q", os.Args[4]) + os.Exit(1) + } + // just "git-lfs-authenticate REPO OPERATION" - authLine := strings.Split(os.Args[4], " ") + authLine := strings.Split(os.Args[5], " ") if len(authLine) < 13 { fmt.Fprintf(os.Stderr, "bad git-lfs-authenticate line: %s\nargs: %v", authLine, os.Args) } diff --git a/test/cmd/lfs-ssh-proxy-test.go b/test/cmd/lfs-ssh-proxy-test.go new file mode 100644 index 00000000..553194e5 --- /dev/null +++ b/test/cmd/lfs-ssh-proxy-test.go @@ -0,0 +1,9 @@ +// +build testtools + +package main + +import "fmt" + +func main() { + fmt.Println("SSH PROXY TEST called") +} diff --git a/test/test-env.sh b/test/test-env.sh index c5f9d12c..71b62dc6 100755 --- a/test/test-env.sh +++ b/test/test-env.sh @@ -673,7 +673,7 @@ begin_test "env with multiple ssh remotes" SSH=git@git-server.com:user/repo.git Endpoint (other)=https://other-git-server.com/user/repo.git/info/lfs (auth=none) SSH=git@other-git-server.com:user/repo.git -GIT_SSH=ssh-echo' +GIT_SSH=lfs-ssh-echo' contains_same_elements "$expected" "$(git lfs env | grep -e "Endpoint" -e "SSH=")" ) diff --git a/test/test-ssh.sh b/test/test-ssh.sh new file mode 100755 index 00000000..7394d802 --- /dev/null +++ b/test/test-ssh.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +. "test/testlib.sh" + +begin_test "ssh with proxy command in lfs.url" +( + set -e + + reponame="batch-ssh-proxy" + setup_remote_repo "$reponame" + clone_repo "$reponame" "$reponame" + + sshurl="${GITSERVER/http:\/\//ssh://-oProxyCommand=ssh-proxy-test/}/$reponame" + echo $sshurl + git config lfs.url "$sshurl" + + contents="test" + oid="$(calc_oid "$contents")" + git lfs track "*.dat" + printf "$contents" > test.dat + git add .gitattributes test.dat + git commit -m "initial commit" + + git push origin master 2>&1 | tee push.log + if [ "0" -eq "${PIPESTATUS[0]}" ]; then + echo >&2 "fatal: push succeeded" + exit 1 + fi + + grep "got 4 args" push.log + grep "lfs-ssh-echo -- -oProxyCommand" push.log +) +end_test diff --git a/test/testenv.sh b/test/testenv.sh index 96d64b31..01091f55 100644 --- a/test/testenv.sh +++ b/test/testenv.sh @@ -120,7 +120,7 @@ TESTHOME="$REMOTEDIR/home" GIT_CONFIG_NOSYSTEM=1 GIT_TERMINAL_PROMPT=0 -GIT_SSH=ssh-echo +GIT_SSH=lfs-ssh-echo APPVEYOR_REPO_COMMIT_MESSAGE="test: env test should look for GIT_SSH too" export CREDSDIR diff --git a/tools/filetools.go b/tools/filetools.go index cc690e31..8e0bc28a 100644 --- a/tools/filetools.go +++ b/tools/filetools.go @@ -139,14 +139,16 @@ type FastWalkCallback func(parentDir string, info os.FileInfo, err error) // determine absolute path rather than tracking it yourself // * Automatically ignores any .git directories // * Respects .gitignore contents and skips ignored files/dirs -func FastWalkGitRepo(dir string, cb FastWalkCallback) { +// +// rootDir - Absolute path to the top of the repository working directory +func FastWalkGitRepo(rootDir string, cb FastWalkCallback) { // Ignore all git metadata including subrepos excludePaths := []filepathfilter.Pattern{ filepathfilter.NewPattern(".git"), filepathfilter.NewPattern(filepath.Join("**", ".git")), } - fileCh := fastWalkWithExcludeFiles(dir, ".gitignore", excludePaths) + fileCh := fastWalkWithExcludeFiles(rootDir, ".gitignore", excludePaths) for file := range fileCh { cb(file.ParentDir, file.Info, file.Err) } @@ -164,17 +166,20 @@ type fastWalkInfo struct { // fastWalkWithExcludeFiles walks the contents of a dir, respecting // include/exclude patterns and also loading new exlude patterns from files // named excludeFilename in directories walked -func fastWalkWithExcludeFiles(dir, excludeFilename string, +// +// rootDir - Absolute path to the top of the repository working directory +func fastWalkWithExcludeFiles(rootDir, excludeFilename string, excludePaths []filepathfilter.Pattern) <-chan fastWalkInfo { fiChan := make(chan fastWalkInfo, 256) - go fastWalkFromRoot(dir, excludeFilename, excludePaths, fiChan) + go fastWalkFromRoot(rootDir, excludeFilename, excludePaths, fiChan) return fiChan } -func fastWalkFromRoot(dir string, excludeFilename string, +// rootDir - Absolute path to the top of the repository working directory +func fastWalkFromRoot(rootDir string, excludeFilename string, excludePaths []filepathfilter.Pattern, fiChan chan<- fastWalkInfo) { - dirFi, err := os.Stat(dir) + dirFi, err := os.Stat(rootDir) if err != nil { fiChan <- fastWalkInfo{Err: err} return @@ -182,7 +187,7 @@ func fastWalkFromRoot(dir string, excludeFilename string, // This waitgroup will be incremented for each nested goroutine var waitg sync.WaitGroup - fastWalkFileOrDir(filepath.Dir(dir), dirFi, excludeFilename, excludePaths, fiChan, &waitg) + fastWalkFileOrDir(true, rootDir, "", dirFi, excludeFilename, excludePaths, fiChan, &waitg) waitg.Wait() close(fiChan) } @@ -193,26 +198,42 @@ func fastWalkFromRoot(dir string, excludeFilename string, // the excludePaths with its content before (parallel) recursing into contents // Also splits large directories into multiple goroutines. // Increments waitg.Add(1) for each new goroutine launched internally -func fastWalkFileOrDir(parentDir string, itemFi os.FileInfo, excludeFilename string, +// +// rootDir - Absolute path to the top of the repository working directory +// workDir - Relative path inside the repository +func fastWalkFileOrDir(isRoot bool, rootDir, workDir string, itemFi os.FileInfo, excludeFilename string, excludePaths []filepathfilter.Pattern, fiChan chan<- fastWalkInfo, waitg *sync.WaitGroup) { - fullPath := filepath.Join(parentDir, itemFi.Name()) + var fullPath string // Absolute path to the current file or dir + var parentWorkDir string // Absolute path to the workDir inside the repository + if isRoot { + fullPath = rootDir + } else { + parentWorkDir = filepath.Join(rootDir, workDir) + fullPath = filepath.Join(parentWorkDir, itemFi.Name()) + } - if !filepathfilter.NewFromPatterns(nil, excludePaths).Allows(fullPath) { + workPath := filepath.Join(workDir, itemFi.Name()) + if !filepathfilter.NewFromPatterns(nil, excludePaths).Allows(workPath) { return } - fiChan <- fastWalkInfo{ParentDir: parentDir, Info: itemFi} + fiChan <- fastWalkInfo{ParentDir: parentWorkDir, Info: itemFi} if !itemFi.IsDir() { // Nothing more to do if this is not a dir return } + var childWorkDir string + if !isRoot { + childWorkDir = filepath.Join(workDir, itemFi.Name()) + } + if len(excludeFilename) > 0 { possibleExcludeFile := filepath.Join(fullPath, excludeFilename) var err error - excludePaths, err = loadExcludeFilename(possibleExcludeFile, fullPath, excludePaths) + excludePaths, err = loadExcludeFilename(possibleExcludeFile, childWorkDir, excludePaths) if err != nil { fiChan <- fastWalkInfo{Err: err} } @@ -231,12 +252,13 @@ func fastWalkFileOrDir(parentDir string, itemFi os.FileInfo, excludeFilename str // The number of items in a dir we process in each goroutine jobSize := 100 + for children, err := df.Readdir(jobSize); err == nil; children, err = df.Readdir(jobSize) { // Parallelise all dirs, and chop large dirs into batches waitg.Add(1) go func(subitems []os.FileInfo) { for _, childFi := range subitems { - fastWalkFileOrDir(fullPath, childFi, excludeFilename, excludePaths, fiChan, waitg) + fastWalkFileOrDir(false, rootDir, childWorkDir, childFi, excludeFilename, excludePaths, fiChan, waitg) } waitg.Done() }(children) @@ -251,7 +273,7 @@ func fastWalkFileOrDir(parentDir string, itemFi os.FileInfo, excludeFilename str // revised array of exclude paths if there are any changes. // If any changes are made a copy of the array is taken so the original is not // modified -func loadExcludeFilename(filename, parentDir string, excludePaths []filepathfilter.Pattern) ([]filepathfilter.Pattern, error) { +func loadExcludeFilename(filename, workDir string, excludePaths []filepathfilter.Pattern) ([]filepathfilter.Pattern, error) { f, err := os.OpenFile(filename, os.O_RDONLY, 0644) if err != nil { if os.IsNotExist(err) { @@ -284,7 +306,7 @@ func loadExcludeFilename(filename, parentDir string, excludePaths []filepathfilt // Allow for both styles of separator at this point if strings.ContainsAny(path, "/\\") || !strings.Contains(path, "*") { - path = filepath.Join(parentDir, line) + path = filepath.Join(workDir, line) } retPaths = append(retPaths, filepathfilter.NewPattern(path)) } diff --git a/tools/filetools_test.go b/tools/filetools_test.go index f78a3351..ade9a492 100644 --- a/tools/filetools_test.go +++ b/tools/filetools_test.go @@ -109,6 +109,9 @@ func TestFastWalkGitRepo(t *testing.T) { "filethatweignore.ign", "foldercontainingignored", "foldercontainingignored/notthisone.ign", + "foldercontainingignored/ignoredfolder", + "foldercontainingignored/ignoredfolder/file1.txt", + "foldercontainingignored/ignoredfolder/file2.txt", "ignoredfolder", "ignoredfolder/file1.txt", "ignoredfolder/file2.txt",