From 9291213c1c19da67bf44365a354b46c31d60d03c Mon Sep 17 00:00:00 2001 From: Rick Olson Date: Fri, 27 Mar 2015 12:14:33 -0600 Subject: [PATCH 1/6] tweak release script temporarily --- script/release.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release.go b/script/release.go index 496f9571..f7bc5e2d 100644 --- a/script/release.go +++ b/script/release.go @@ -12,7 +12,7 @@ import ( var ( ReleaseId = flag.Int("id", 0, "github/git-lfs Release ID") - uploadUrlFmt = "https://uploads.github.com/repos/github/git-lfs/releases/%d/assets?%s" + uploadUrlFmt = "https://uploads.github.com/repos/github/git-lfs-interim/releases/%d/assets?%s" ) func mainRelease() { From f131898b77c801d819487a3c2c6c356300efa255 Mon Sep 17 00:00:00 2001 From: Rick Olson Date: Fri, 27 Mar 2015 16:21:08 -0600 Subject: [PATCH 2/6] rename these test methods --- lfs/config_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lfs/config_test.go b/lfs/config_test.go index c758b431..26697f4a 100644 --- a/lfs/config_test.go +++ b/lfs/config_test.go @@ -52,7 +52,7 @@ func TestEndpointUseAlternateRemote(t *testing.T) { assert.Equal(t, "def", config.Endpoint()) } -func TestEndpointAddsMediaSuffix(t *testing.T) { +func TestEndpointAddsLfsSuffix(t *testing.T) { config := &Configuration{ gitConfig: map[string]string{"remote.origin.url": "https://example.com/foo/bar"}, remotes: []string{}, @@ -61,7 +61,7 @@ func TestEndpointAddsMediaSuffix(t *testing.T) { assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", config.Endpoint()) } -func TestBareEndpointAddsMediaSuffix(t *testing.T) { +func TestBareEndpointAddsLfsSuffix(t *testing.T) { config := &Configuration{ gitConfig: map[string]string{"remote.origin.url": "https://example.com/foo/bar.git"}, remotes: []string{}, @@ -70,7 +70,7 @@ func TestBareEndpointAddsMediaSuffix(t *testing.T) { assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", config.Endpoint()) } -func TestSSHEndpointAddsMediaSuffix(t *testing.T) { +func TestSSHEndpointAddsLfsSuffix(t *testing.T) { config := &Configuration{ gitConfig: map[string]string{"remote.origin.url": "git@example.com:foo/bar"}, remotes: []string{}, @@ -79,7 +79,7 @@ func TestSSHEndpointAddsMediaSuffix(t *testing.T) { assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", config.Endpoint()) } -func TestBareSSHEndpointAddsMediaSuffix(t *testing.T) { +func TestBareSSHEndpointAddsLfsSuffix(t *testing.T) { config := &Configuration{ gitConfig: map[string]string{"remote.origin.url": "git@example.com:foo/bar.git"}, remotes: []string{}, @@ -88,7 +88,7 @@ func TestBareSSHEndpointAddsMediaSuffix(t *testing.T) { assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", config.Endpoint()) } -func TestHTTPEndpointAddsMediaSuffix(t *testing.T) { +func TestHTTPEndpointAddsLfsSuffix(t *testing.T) { config := &Configuration{ gitConfig: map[string]string{"remote.origin.url": "http://example.com/foo/bar"}, remotes: []string{}, @@ -97,7 +97,7 @@ func TestHTTPEndpointAddsMediaSuffix(t *testing.T) { assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", config.Endpoint()) } -func TestBareHTTPEndpointAddsMediaSuffix(t *testing.T) { +func TestBareHTTPEndpointAddsLfsSuffix(t *testing.T) { config := &Configuration{ gitConfig: map[string]string{"remote.origin.url": "http://example.com/foo/bar.git"}, remotes: []string{}, From 514d947c8e97c8e96a558b8b1d6a52009f12c1a6 Mon Sep 17 00:00:00 2001 From: Rick Olson Date: Fri, 27 Mar 2015 16:36:33 -0600 Subject: [PATCH 3/6] teach config.Endpoint() to return an Endpoint struct with optional ssh info $ git lfs env Endpoint=https://172.28.128.4/ghe-admin/git-lfs-test.git/info/lfs SSH=git@172.28.128.4:ghe-admin/git-lfs-test.git LocalWorkingDir=/Users/rick/github/git-lfs-test LocalGitDir=/Users/rick/github/git-lfs-test/.git LocalMediaDir=/Users/rick/github/git-lfs-test/.git/lfs/objects TempDir=/Users/rick/github/git-lfs-test/.git/lfs/tmp --- commands/command_env.go | 15 ++++++++-- lfs/client.go | 2 +- lfs/config.go | 37 ++++++++++++++++------- lfs/config_test.go | 65 ++++++++++++++++++++++++++++++++++------- 4 files changed, 94 insertions(+), 25 deletions(-) diff --git a/commands/command_env.go b/commands/command_env.go index 95d17a94..459449e1 100644 --- a/commands/command_env.go +++ b/commands/command_env.go @@ -16,12 +16,21 @@ var ( func envCommand(cmd *cobra.Command, args []string) { config := lfs.Config - if endpoint := config.Endpoint(); len(endpoint) > 0 { - Print("Endpoint=%s", endpoint) + endpoint := config.Endpoint() + + if len(endpoint.Url) > 0 { + Print("Endpoint=%s", endpoint.Url) + if len(endpoint.SshUserAndHost) > 0 { + Print(" SSH=%s:%s", endpoint.SshUserAndHost, endpoint.SshPath) + } } for _, remote := range config.Remotes() { - Print("Endpoint (%s)=%s", remote, config.RemoteEndpoint(remote)) + remoteEndpoint := config.RemoteEndpoint(remote) + Print("Endpoint (%s)=%s", remote, remoteEndpoint.Url) + if len(endpoint.SshUserAndHost) > 0 { + Print(" SSH=%s:%s", endpoint.SshUserAndHost, endpoint.SshPath) + } } for _, env := range lfs.Environ() { diff --git a/lfs/client.go b/lfs/client.go index 42c28753..e7a75f92 100644 --- a/lfs/client.go +++ b/lfs/client.go @@ -417,7 +417,7 @@ func getCreds(req *http.Request) (Creds, error) { } func setErrorRequestContext(err *WrappedError, req *http.Request) { - err.Set("Endpoint", Config.Endpoint()) + err.Set("Endpoint", Config.Endpoint().Url) err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String())) setErrorHeaderContext(err, "Response", req.Header) } diff --git a/lfs/config.go b/lfs/config.go index e2b47629..cab83f67 100644 --- a/lfs/config.go +++ b/lfs/config.go @@ -20,6 +20,12 @@ type Configuration struct { isTracingHttp bool } +type Endpoint struct { + Url string + SshUserAndHost string + SshPath string +} + var ( Config = NewConfig() httpPrefixRe = regexp.MustCompile("\\Ahttps?://") @@ -34,13 +40,13 @@ func NewConfig() *Configuration { return c } -func (c *Configuration) Endpoint() string { +func (c *Configuration) Endpoint() Endpoint { if url, ok := c.GitConfig("lfs.url"); ok { - return url + return Endpoint{Url: url} } if len(c.CurrentRemote) > 0 && c.CurrentRemote != defaultRemote { - if endpoint := c.RemoteEndpoint(c.CurrentRemote); len(endpoint) > 0 { + if endpoint := c.RemoteEndpoint(c.CurrentRemote); len(endpoint.Url) > 0 { return endpoint } } @@ -48,32 +54,41 @@ func (c *Configuration) Endpoint() string { return c.RemoteEndpoint(defaultRemote) } -func (c *Configuration) RemoteEndpoint(remote string) string { +func (c *Configuration) RemoteEndpoint(remote string) Endpoint { if len(remote) == 0 { remote = defaultRemote } if url, ok := c.GitConfig("remote." + remote + ".lfs_url"); ok { - return url + return Endpoint{Url: url} } if url, ok := c.GitConfig("remote." + remote + ".url"); ok { + endpoint := Endpoint{Url: url} + if !httpPrefixRe.MatchString(url) { pieces := strings.SplitN(url, ":", 2) hostPieces := strings.SplitN(pieces[0], "@", 2) if len(hostPieces) < 2 { - return "unknown" + endpoint.Url = "" + return endpoint } - url = fmt.Sprintf("https://%s/%s", hostPieces[1], pieces[1]) + + endpoint.SshUserAndHost = pieces[0] + endpoint.SshPath = pieces[1] + endpoint.Url = fmt.Sprintf("https://%s/%s", hostPieces[1], pieces[1]) } if path.Ext(url) == ".git" { - return url + "/info/lfs" + endpoint.Url += "/info/lfs" + } else { + endpoint.Url += ".git/info/lfs" } - return url + ".git/info/lfs" + + return endpoint } - return "" + return Endpoint{} } func (c *Configuration) Remotes() []string { @@ -93,7 +108,7 @@ func (c *Configuration) SetConfig(key, value string) { } func (c *Configuration) ObjectUrl(oid string) (*url.URL, error) { - u, err := url.Parse(c.Endpoint()) + u, err := url.Parse(c.Endpoint().Url) if err != nil { return nil, err } diff --git a/lfs/config_test.go b/lfs/config_test.go index 26697f4a..7a2f4546 100644 --- a/lfs/config_test.go +++ b/lfs/config_test.go @@ -11,7 +11,10 @@ func TestEndpointDefaultsToOrigin(t *testing.T) { remotes: []string{}, } - assert.Equal(t, "abc", config.Endpoint()) + endpoint := config.Endpoint() + assert.Equal(t, "abc", endpoint.Url) + assert.Equal(t, "", endpoint.SshUserAndHost) + assert.Equal(t, "", endpoint.SshPath) } func TestEndpointOverridesOrigin(t *testing.T) { @@ -23,7 +26,10 @@ func TestEndpointOverridesOrigin(t *testing.T) { remotes: []string{}, } - assert.Equal(t, "abc", config.Endpoint()) + endpoint := config.Endpoint() + assert.Equal(t, "abc", endpoint.Url) + assert.Equal(t, "", endpoint.SshUserAndHost) + assert.Equal(t, "", endpoint.SshPath) } func TestEndpointNoOverrideDefaultRemote(t *testing.T) { @@ -35,7 +41,10 @@ func TestEndpointNoOverrideDefaultRemote(t *testing.T) { remotes: []string{}, } - assert.Equal(t, "abc", config.Endpoint()) + endpoint := config.Endpoint() + assert.Equal(t, "abc", endpoint.Url) + assert.Equal(t, "", endpoint.SshUserAndHost) + assert.Equal(t, "", endpoint.SshPath) } func TestEndpointUseAlternateRemote(t *testing.T) { @@ -49,7 +58,10 @@ func TestEndpointUseAlternateRemote(t *testing.T) { config.CurrentRemote = "other" - assert.Equal(t, "def", config.Endpoint()) + endpoint := config.Endpoint() + assert.Equal(t, "def", endpoint.Url) + assert.Equal(t, "", endpoint.SshUserAndHost) + assert.Equal(t, "", endpoint.SshPath) } func TestEndpointAddsLfsSuffix(t *testing.T) { @@ -58,7 +70,10 @@ func TestEndpointAddsLfsSuffix(t *testing.T) { remotes: []string{}, } - assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", config.Endpoint()) + endpoint := config.Endpoint() + assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url) + assert.Equal(t, "", endpoint.SshUserAndHost) + assert.Equal(t, "", endpoint.SshPath) } func TestBareEndpointAddsLfsSuffix(t *testing.T) { @@ -67,7 +82,25 @@ func TestBareEndpointAddsLfsSuffix(t *testing.T) { remotes: []string{}, } - assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", config.Endpoint()) + endpoint := config.Endpoint() + assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url) + assert.Equal(t, "", endpoint.SshUserAndHost) + assert.Equal(t, "", endpoint.SshPath) +} + +func TestSSHEndpointOverridden(t *testing.T) { + config := &Configuration{ + gitConfig: map[string]string{ + "remote.origin.url": "git@example.com:foo/bar", + "remote.origin.lfs_url": "lfs", + }, + remotes: []string{}, + } + + endpoint := config.Endpoint() + assert.Equal(t, "lfs", endpoint.Url) + assert.Equal(t, "", endpoint.SshUserAndHost) + assert.Equal(t, "", endpoint.SshPath) } func TestSSHEndpointAddsLfsSuffix(t *testing.T) { @@ -76,7 +109,10 @@ func TestSSHEndpointAddsLfsSuffix(t *testing.T) { remotes: []string{}, } - assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", config.Endpoint()) + endpoint := config.Endpoint() + assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url) + assert.Equal(t, "git@example.com", endpoint.SshUserAndHost) + assert.Equal(t, "foo/bar", endpoint.SshPath) } func TestBareSSHEndpointAddsLfsSuffix(t *testing.T) { @@ -85,7 +121,10 @@ func TestBareSSHEndpointAddsLfsSuffix(t *testing.T) { remotes: []string{}, } - assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", config.Endpoint()) + endpoint := config.Endpoint() + assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url) + assert.Equal(t, "git@example.com", endpoint.SshUserAndHost) + assert.Equal(t, "foo/bar.git", endpoint.SshPath) } func TestHTTPEndpointAddsLfsSuffix(t *testing.T) { @@ -94,7 +133,10 @@ func TestHTTPEndpointAddsLfsSuffix(t *testing.T) { remotes: []string{}, } - assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", config.Endpoint()) + endpoint := config.Endpoint() + assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", endpoint.Url) + assert.Equal(t, "", endpoint.SshUserAndHost) + assert.Equal(t, "", endpoint.SshPath) } func TestBareHTTPEndpointAddsLfsSuffix(t *testing.T) { @@ -103,7 +145,10 @@ func TestBareHTTPEndpointAddsLfsSuffix(t *testing.T) { remotes: []string{}, } - assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", config.Endpoint()) + endpoint := config.Endpoint() + assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", endpoint.Url) + assert.Equal(t, "", endpoint.SshUserAndHost) + assert.Equal(t, "", endpoint.SshPath) } func TestObjectUrl(t *testing.T) { From 9c67ba5a58688bd2a4e9d3c0642040c59c5a6d58 Mon Sep 17 00:00:00 2001 From: Rick Olson Date: Sun, 29 Mar 2015 13:16:28 -0600 Subject: [PATCH 4/6] promote ObjectUrl to a package level function --- lfs/config.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/lfs/config.go b/lfs/config.go index cab83f67..411eab14 100644 --- a/lfs/config.go +++ b/lfs/config.go @@ -40,6 +40,19 @@ func NewConfig() *Configuration { return c } +func ObjectUrl(endpoint Endpoint, oid string) (*url.URL, error) { + u, err := url.Parse(endpoint.Url) + if err != nil { + return nil, err + } + + u.Path = path.Join(u.Path, "objects") + if len(oid) > 0 { + u.Path = path.Join(u.Path, oid) + } + return u, nil +} + func (c *Configuration) Endpoint() Endpoint { if url, ok := c.GitConfig("lfs.url"); ok { return Endpoint{Url: url} @@ -108,16 +121,7 @@ func (c *Configuration) SetConfig(key, value string) { } func (c *Configuration) ObjectUrl(oid string) (*url.URL, error) { - u, err := url.Parse(c.Endpoint().Url) - if err != nil { - return nil, err - } - - u.Path = path.Join(u.Path, "objects") - if len(oid) > 0 { - u.Path = path.Join(u.Path, oid) - } - return u, nil + return ObjectUrl(c.Endpoint(), oid) } type AltConfig struct { From 6b95a6e40b9b22d5cf9d89465d10752a437ebf6c Mon Sep 17 00:00:00 2001 From: Rick Olson Date: Sun, 29 Mar 2015 17:13:50 -0600 Subject: [PATCH 5/6] merge ssh header if the git remote is an ssh endpoint --- lfs/client.go | 13 ++++++++++-- lfs/ssh.go | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 lfs/ssh.go diff --git a/lfs/client.go b/lfs/client.go index e7a75f92..7053a6f0 100644 --- a/lfs/client.go +++ b/lfs/client.go @@ -148,7 +148,7 @@ func Upload(oidPath, filename string, cb CopyCallback) *WrappedError { return Error(err) } - req, creds, err := newApiRequest("POST", "") + req, creds, err := newApiRequest("POST", oid) if err != nil { return Error(err) } @@ -367,7 +367,15 @@ func saveCredentials(creds Creds, res *http.Response) { } func newApiRequest(method, oid string) (*http.Request, Creds, error) { - u, err := Config.ObjectUrl(oid) + endpoint := Config.Endpoint() + objectOid := oid + operation := "download" + if method == "POST" { + objectOid = "" + operation = "upload" + } + + u, err := ObjectUrl(endpoint, objectOid) if err != nil { return nil, nil, err } @@ -375,6 +383,7 @@ func newApiRequest(method, oid string) (*http.Request, Creds, error) { req, creds, err := newClientRequest(method, u.String()) if err == nil { req.Header.Set("Accept", mediaType) + err = mergeSshHeader(req.Header, endpoint, operation, oid) } return req, creds, err } diff --git a/lfs/ssh.go b/lfs/ssh.go new file mode 100644 index 00000000..b1db2703 --- /dev/null +++ b/lfs/ssh.go @@ -0,0 +1,56 @@ +package lfs + +import ( + "encoding/json" + "github.com/rubyist/tracerx" + "net/http" + "os/exec" +) + +type sshAuthResponse struct { + Message string `json:"-"` + Header map[string]string `json:"header"` + ExpiresAt string `json:"expires_at"` +} + +func mergeSshHeader(header http.Header, endpoint Endpoint, operation, oid string) error { + tracerx.Printf("endpoint: %s / %s", endpoint.SshUserAndHost, endpoint.SshPath) + if len(endpoint.SshUserAndHost) == 0 { + return nil + } + + res, err := sshAuthenticate(endpoint, operation, oid) + if err != nil { + return err + } + + if res.Header != nil { + for key, value := range res.Header { + header.Set(key, value) + } + } + + return nil +} + +func sshAuthenticate(endpoint Endpoint, operation, oid string) (sshAuthResponse, error) { + tracerx.Printf("ssh %s git-lfs-authenticate %s %s %s", + endpoint.SshUserAndHost, endpoint.SshPath, operation, oid) + cmd := exec.Command("ssh", endpoint.SshUserAndHost, + "git-lfs-authenticate", + endpoint.SshPath, + operation, oid, + ) + + out, err := cmd.CombinedOutput() + tracerx.Printf("ssh: %s", string(out)) + res := sshAuthResponse{} + + if err != nil { + res.Message = string(out) + } else { + err = json.Unmarshal(out, &res) + } + + return res, err +} From f417fa5ee691b37542574b414fdfb6d3a3d94ad2 Mon Sep 17 00:00:00 2001 From: Rick Olson Date: Mon, 30 Mar 2015 00:00:34 -0600 Subject: [PATCH 6/6] print ssh error to tracerx so it can try hitting the api directly --- lfs/client.go | 6 +++++- lfs/ssh.go | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lfs/client.go b/lfs/client.go index 7053a6f0..be0a3c80 100644 --- a/lfs/client.go +++ b/lfs/client.go @@ -383,7 +383,11 @@ func newApiRequest(method, oid string) (*http.Request, Creds, error) { req, creds, err := newClientRequest(method, u.String()) if err == nil { req.Header.Set("Accept", mediaType) - err = mergeSshHeader(req.Header, endpoint, operation, oid) + if err := mergeSshHeader(req.Header, endpoint, operation, oid); err != nil { + tracerx.Printf("ssh: attempted with %s. Error: %s", + endpoint.SshUserAndHost, err.Error(), + ) + } } return req, creds, err } diff --git a/lfs/ssh.go b/lfs/ssh.go index b1db2703..655201b3 100644 --- a/lfs/ssh.go +++ b/lfs/ssh.go @@ -14,7 +14,6 @@ type sshAuthResponse struct { } func mergeSshHeader(header http.Header, endpoint Endpoint, operation, oid string) error { - tracerx.Printf("endpoint: %s / %s", endpoint.SshUserAndHost, endpoint.SshPath) if len(endpoint.SshUserAndHost) == 0 { return nil } @@ -34,7 +33,7 @@ func mergeSshHeader(header http.Header, endpoint Endpoint, operation, oid string } func sshAuthenticate(endpoint Endpoint, operation, oid string) (sshAuthResponse, error) { - tracerx.Printf("ssh %s git-lfs-authenticate %s %s %s", + tracerx.Printf("ssh: %s git-lfs-authenticate %s %s %s", endpoint.SshUserAndHost, endpoint.SshPath, operation, oid) cmd := exec.Command("ssh", endpoint.SshUserAndHost, "git-lfs-authenticate", @@ -43,7 +42,6 @@ func sshAuthenticate(endpoint Endpoint, operation, oid string) (sshAuthResponse, ) out, err := cmd.CombinedOutput() - tracerx.Printf("ssh: %s", string(out)) res := sshAuthResponse{} if err != nil {