From 1ba67ca09c87e39d0100d95524163e5ce05345c2 Mon Sep 17 00:00:00 2001 From: risk danger olson Date: Tue, 20 Dec 2016 10:29:26 -0700 Subject: [PATCH] lfsapi: add proxy support --- lfsapi/auth_test.go | 6 +- lfsapi/endpoint_finder_test.go | 78 ++++++++--------------- lfsapi/lfsapi.go | 81 ++++++++++++++++++++++- lfsapi/proxy.go | 113 +++++++++++++++++++++++++++++++++ lfsapi/proxy_test.go | 82 ++++++++++++++++++++++++ 5 files changed, 303 insertions(+), 57 deletions(-) create mode 100644 lfsapi/proxy.go create mode 100644 lfsapi/proxy_test.go diff --git a/lfsapi/auth_test.go b/lfsapi/auth_test.go index 69511a60..dbbf8d12 100644 --- a/lfsapi/auth_test.go +++ b/lfsapi/auth_test.go @@ -63,7 +63,7 @@ func TestDoWithAuthApprove(t *testing.T) { creds := newMockCredentialHelper() c := &Client{ Credentials: creds, - Endpoints: NewEndpointFinder(gitEnv(map[string]string{ + Endpoints: NewEndpointFinder(testEnv(map[string]string{ "lfs.url": srv.URL, })), } @@ -125,7 +125,7 @@ func TestDoWithAuthReject(t *testing.T) { c := &Client{ Credentials: creds, - Endpoints: NewEndpointFinder(gitEnv(map[string]string{ + Endpoints: NewEndpointFinder(testEnv(map[string]string{ "lfs.url": srv.URL, })), } @@ -343,7 +343,7 @@ func TestGetCredentials(t *testing.T) { for _, check := range checks { t.Logf("Checking %q", check.Desc) - ef := NewEndpointFinder(gitEnv(check.Config)) + ef := NewEndpointFinder(testEnv(check.Config)) req, err := http.NewRequest(check.Method, check.Href, nil) if err != nil { diff --git a/lfsapi/endpoint_finder_test.go b/lfsapi/endpoint_finder_test.go index 4b2babe1..0663dc09 100644 --- a/lfsapi/endpoint_finder_test.go +++ b/lfsapi/endpoint_finder_test.go @@ -1,14 +1,13 @@ package lfsapi import ( - "strings" "testing" "github.com/stretchr/testify/assert" ) func TestEndpointDefaultsToOrigin(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.lfsurl": "abc", })) @@ -19,7 +18,7 @@ func TestEndpointDefaultsToOrigin(t *testing.T) { } func TestEndpointOverridesOrigin(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "lfs.url": "abc", "remote.origin.lfsurl": "def", })) @@ -31,7 +30,7 @@ func TestEndpointOverridesOrigin(t *testing.T) { } func TestEndpointNoOverrideDefaultRemote(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.lfsurl": "abc", "remote.other.lfsurl": "def", })) @@ -43,7 +42,7 @@ func TestEndpointNoOverrideDefaultRemote(t *testing.T) { } func TestEndpointUseAlternateRemote(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.lfsurl": "abc", "remote.other.lfsurl": "def", })) @@ -55,7 +54,7 @@ func TestEndpointUseAlternateRemote(t *testing.T) { } func TestEndpointAddsLfsSuffix(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "https://example.com/foo/bar", })) @@ -66,7 +65,7 @@ func TestEndpointAddsLfsSuffix(t *testing.T) { } func TestBareEndpointAddsLfsSuffix(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "https://example.com/foo/bar.git", })) @@ -77,7 +76,7 @@ func TestBareEndpointAddsLfsSuffix(t *testing.T) { } func TestEndpointSeparateClonePushUrl(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "https://example.com/foo/bar.git", "remote.origin.pushurl": "https://readwrite.com/foo/bar.git", })) @@ -94,7 +93,7 @@ func TestEndpointSeparateClonePushUrl(t *testing.T) { } func TestEndpointOverriddenSeparateClonePushLfsUrl(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "https://example.com/foo/bar.git", "remote.origin.pushurl": "https://readwrite.com/foo/bar.git", "remote.origin.lfsurl": "https://examplelfs.com/foo/bar", @@ -113,7 +112,7 @@ func TestEndpointOverriddenSeparateClonePushLfsUrl(t *testing.T) { } func TestEndpointGlobalSeparateLfsPush(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "lfs.url": "https://readonly.com/foo/bar", "lfs.pushurl": "https://write.com/foo/bar", })) @@ -130,7 +129,7 @@ func TestEndpointGlobalSeparateLfsPush(t *testing.T) { } func TestSSHEndpointOverridden(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "git@example.com:foo/bar", "remote.origin.lfsurl": "lfs", })) @@ -143,7 +142,7 @@ func TestSSHEndpointOverridden(t *testing.T) { } func TestSSHEndpointAddsLfsSuffix(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "ssh://git@example.com/foo/bar", })) @@ -155,7 +154,7 @@ func TestSSHEndpointAddsLfsSuffix(t *testing.T) { } func TestSSHCustomPortEndpointAddsLfsSuffix(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "ssh://git@example.com:9000/foo/bar", })) @@ -167,7 +166,7 @@ func TestSSHCustomPortEndpointAddsLfsSuffix(t *testing.T) { } func TestBareSSHEndpointAddsLfsSuffix(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "git@example.com:foo/bar.git", })) @@ -179,7 +178,7 @@ func TestBareSSHEndpointAddsLfsSuffix(t *testing.T) { } func TestSSHEndpointFromGlobalLfsUrl(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "lfs.url": "git@example.com:foo/bar.git", })) @@ -191,7 +190,7 @@ func TestSSHEndpointFromGlobalLfsUrl(t *testing.T) { } func TestHTTPEndpointAddsLfsSuffix(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "http://example.com/foo/bar", })) @@ -203,7 +202,7 @@ func TestHTTPEndpointAddsLfsSuffix(t *testing.T) { } func TestBareHTTPEndpointAddsLfsSuffix(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "http://example.com/foo/bar.git", })) @@ -215,7 +214,7 @@ func TestBareHTTPEndpointAddsLfsSuffix(t *testing.T) { } func TestGitEndpointAddsLfsSuffix(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "git://example.com/foo/bar", })) @@ -227,7 +226,7 @@ func TestGitEndpointAddsLfsSuffix(t *testing.T) { } func TestGitEndpointAddsLfsSuffixWithCustomProtocol(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "git://example.com/foo/bar", "lfs.gitprotocol": "http", })) @@ -240,7 +239,7 @@ func TestGitEndpointAddsLfsSuffixWithCustomProtocol(t *testing.T) { } func TestBareGitEndpointAddsLfsSuffix(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "remote.origin.url": "git://example.com/foo/bar.git", })) @@ -267,7 +266,7 @@ func TestAccessConfig(t *testing.T) { } for value, expected := range tests { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "lfs.url": "http://example.com", "lfs.http://example.com.access": value, "lfs.https://example.com.access": "bad", @@ -286,7 +285,7 @@ func TestAccessConfig(t *testing.T) { // Test again but with separate push url for value, expected := range tests { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "lfs.url": "http://example.com", "lfs.pushurl": "http://examplepush.com", "lfs.http://example.com.access": value, @@ -313,7 +312,7 @@ func TestAccessAbsentConfig(t *testing.T) { } func TestSetAccess(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{})) + finder := NewEndpointFinder(testEnv(map[string]string{})) assert.Equal(t, NoneAccess, finder.AccessFor("http://example.com")) finder.SetAccess("http://example.com", NTLMAccess) @@ -321,7 +320,7 @@ func TestSetAccess(t *testing.T) { } func TestChangeAccess(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "lfs.http://example.com.access": "basic", })) @@ -331,7 +330,7 @@ func TestChangeAccess(t *testing.T) { } func TestDeleteAccessWithNone(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "lfs.http://example.com.access": "basic", })) @@ -341,7 +340,7 @@ func TestDeleteAccessWithNone(t *testing.T) { } func TestDeleteAccessWithEmptyString(t *testing.T) { - finder := NewEndpointFinder(gitEnv(map[string]string{ + finder := NewEndpointFinder(testEnv(map[string]string{ "lfs.http://example.com.access": "basic", })) @@ -349,30 +348,3 @@ func TestDeleteAccessWithEmptyString(t *testing.T) { finder.SetAccess("http://example.com", Access("")) assert.Equal(t, NoneAccess, finder.AccessFor("http://example.com")) } - -type gitEnv map[string]string - -func (e gitEnv) Get(key string) (string, bool) { - v, ok := e[key] - return v, ok -} - -func (e gitEnv) Bool(key string, def bool) (val bool) { - s, _ := e.Get(key) - if len(s) == 0 { - return def - } - - switch strings.ToLower(s) { - case "true", "1", "on", "yes", "t": - return true - case "false", "0", "off", "no", "f": - return false - default: - return false - } -} - -func (e gitEnv) All() map[string]string { - return e -} diff --git a/lfsapi/lfsapi.go b/lfsapi/lfsapi.go index 3e66b323..b75afcd4 100644 --- a/lfsapi/lfsapi.go +++ b/lfsapi/lfsapi.go @@ -5,6 +5,7 @@ import ( "net" "net/http" "regexp" + "strings" "sync" "time" @@ -26,22 +27,39 @@ type Client struct { TLSTimeout int `git:"lfs.tlstimeout"` ConcurrentTransfers int `git:"lfs.concurrenttransfers"` + HTTPSProxy string + HTTPProxy string + NoProxy string + hostClients map[string]*http.Client clientMu sync.Mutex } func NewClient(osEnv env, gitEnv env) (*Client, error) { + if osEnv == nil { + osEnv = make(testEnv) + } + + if gitEnv == nil { + gitEnv = make(testEnv) + } + netrc, err := ParseNetrc(osEnv) if err != nil { return nil, err } + httpsProxy, httpProxy, noProxy := getProxyServers(osEnv, gitEnv) + return &Client{ Endpoints: NewEndpointFinder(gitEnv), Credentials: &CommandCredentialHelper{ SkipPrompt: !osEnv.Bool("GIT_TERMINAL_PROMPT", true), }, - Netrc: netrc, + Netrc: netrc, + HTTPSProxy: httpsProxy, + HTTPProxy: httpProxy, + NoProxy: noProxy, }, nil } @@ -87,6 +105,7 @@ func (c *Client) httpClient(host string) *http.Client { } tr := &http.Transport{ + Proxy: ProxyFromClient(c), Dial: (&net.Dialer{ Timeout: time.Duration(dialtime) * time.Second, KeepAlive: time.Duration(keepalivetime) * time.Second, @@ -104,6 +123,37 @@ func (c *Client) httpClient(host string) *http.Client { return httpClient } +func getProxyServers(osEnv env, gitEnv env) (string, string, string) { + var httpsProxy string + httpProxy, _ := gitEnv.Get("http.proxy") + if strings.HasPrefix(httpProxy, "https://") { + httpsProxy = httpProxy + } + + if len(httpsProxy) == 0 { + httpsProxy, _ = osEnv.Get("HTTPS_PROXY") + } + + if len(httpsProxy) == 0 { + httpsProxy, _ = osEnv.Get("https_proxy") + } + + if len(httpProxy) == 0 { + httpProxy, _ = osEnv.Get("HTTP_PROXY") + } + + if len(httpProxy) == 0 { + httpProxy, _ = osEnv.Get("http_proxy") + } + + noProxy, _ := osEnv.Get("NO_PROXY") + if len(noProxy) == 0 { + noProxy, _ = osEnv.Get("no_proxy") + } + + return httpsProxy, httpProxy, noProxy +} + func decodeResponse(res *http.Response, obj interface{}) error { ctype := res.Header.Get("Content-Type") if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) { @@ -119,3 +169,32 @@ func decodeResponse(res *http.Response, obj interface{}) error { return nil } + +// basic config.Environment implementation. Only used in tests, or as a zero +// value to NewClient(). +type testEnv map[string]string + +func (e testEnv) Get(key string) (string, bool) { + v, ok := e[key] + return v, ok +} + +func (e testEnv) Bool(key string, def bool) (val bool) { + s, _ := e.Get(key) + if len(s) == 0 { + return def + } + + switch strings.ToLower(s) { + case "true", "1", "on", "yes", "t": + return true + case "false", "0", "off", "no", "f": + return false + default: + return false + } +} + +func (e testEnv) All() map[string]string { + return e +} diff --git a/lfsapi/proxy.go b/lfsapi/proxy.go new file mode 100644 index 00000000..29549b41 --- /dev/null +++ b/lfsapi/proxy.go @@ -0,0 +1,113 @@ +package lfsapi + +import ( + "net/http" + "net/url" + "strings" + + "fmt" +) + +// Logic is copied, with small changes, from "net/http".ProxyFromEnvironment in the go std lib. +func ProxyFromClient(c *Client) func(req *http.Request) (*url.URL, error) { + httpProxy := c.HTTPProxy + httpsProxy := c.HTTPSProxy + noProxy := c.NoProxy + + return func(req *http.Request) (*url.URL, error) { + var proxy string + if req.URL.Scheme == "https" { + proxy = httpsProxy + } + + if len(proxy) == 0 { + proxy = httpProxy + } + + if len(proxy) == 0 { + return nil, nil + } + + if !useProxy(noProxy, canonicalAddr(req.URL)) { + return nil, nil + } + + proxyURL, err := url.Parse(proxy) + if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") { + // proxy was bogus. Try prepending "http://" to it and + // see if that parses correctly. If not, we fall + // through and complain about the original one. + if httpProxyURL, httpErr := url.Parse("http://" + proxy); httpErr == nil { + return httpProxyURL, nil + } + } + if err != nil { + return nil, fmt.Errorf("invalid proxy address: %q: %v", proxy, err) + } + return proxyURL, nil + } +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +// Copied from "net/http".ProxyFromEnvironment in the go std lib. +func canonicalAddr(url *url.URL) string { + addr := url.Host + if !hasPort(addr) { + return addr + ":" + portMap[url.Scheme] + } + return addr +} + +// useProxy reports whether requests to addr should use a proxy, +// according to the noProxy or noProxy environment variable. +// addr is always a canonicalAddr with a host and port. +// Copied from "net/http".ProxyFromEnvironment in the go std lib +// and adapted to allow proxy usage even for localhost. +func useProxy(noProxy, addr string) bool { + if len(addr) == 0 { + return true + } + + if noProxy == "*" { + return false + } + + addr = strings.ToLower(strings.TrimSpace(addr)) + if hasPort(addr) { + addr = addr[:strings.LastIndex(addr, ":")] + } + + for _, p := range strings.Split(noProxy, ",") { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + if hasPort(p) { + p = p[:strings.LastIndex(p, ":")] + } + if addr == p { + return false + } + if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) { + // noProxy ".foo.com" matches "bar.foo.com" or "foo.com" + return false + } + if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' { + // noProxy "foo.com" matches "bar.foo.com" + return false + } + } + return true +} + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +// Copied from "net/http".ProxyFromEnvironment in the go std lib. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +var ( + portMap = map[string]string{ + "http": "80", + "https": "443", + } +) diff --git a/lfsapi/proxy_test.go b/lfsapi/proxy_test.go new file mode 100644 index 00000000..0b00e4c9 --- /dev/null +++ b/lfsapi/proxy_test.go @@ -0,0 +1,82 @@ +package lfsapi + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProxyFromGitConfig(t *testing.T) { + c, err := NewClient(testEnv(map[string]string{ + "HTTPS_PROXY": "https://proxy-from-env:8080", + }), testEnv(map[string]string{ + "http.proxy": "https://proxy-from-git-config:8080", + })) + require.Nil(t, err) + + req, err := http.NewRequest("GET", "https://some-host.com:123/foo/bar", nil) + require.Nil(t, err) + + proxyURL, err := ProxyFromClient(c)(req) + assert.Equal(t, "proxy-from-git-config:8080", proxyURL.Host) + assert.Nil(t, err) +} + +func TestHttpProxyFromGitConfig(t *testing.T) { + c, err := NewClient(testEnv(map[string]string{ + "HTTPS_PROXY": "https://proxy-from-env:8080", + }), testEnv(map[string]string{ + "http.proxy": "http://proxy-from-git-config:8080", + })) + require.Nil(t, err) + + req, err := http.NewRequest("GET", "https://some-host.com:123/foo/bar", nil) + require.Nil(t, err) + + proxyURL, err := ProxyFromClient(c)(req) + assert.Equal(t, "proxy-from-env:8080", proxyURL.Host) + assert.Nil(t, err) +} + +func TestProxyFromEnvironment(t *testing.T) { + c, err := NewClient(testEnv(map[string]string{ + "HTTPS_PROXY": "https://proxy-from-env:8080", + }), nil) + require.Nil(t, err) + + req, err := http.NewRequest("GET", "https://some-host.com:123/foo/bar", nil) + require.Nil(t, err) + + proxyURL, err := ProxyFromClient(c)(req) + assert.Equal(t, "proxy-from-env:8080", proxyURL.Host) + assert.Nil(t, err) +} + +func TestProxyIsNil(t *testing.T) { + c := &Client{} + + req, err := http.NewRequest("GET", "http://some-host.com:123/foo/bar", nil) + require.Nil(t, err) + + proxyURL, err := ProxyFromClient(c)(req) + assert.Nil(t, proxyURL) + assert.Nil(t, err) +} + +func TestProxyNoProxy(t *testing.T) { + c, err := NewClient(testEnv(map[string]string{ + "NO_PROXY": "some-host", + }), testEnv(map[string]string{ + "http.proxy": "https://proxy-from-git-config:8080", + })) + require.Nil(t, err) + + req, err := http.NewRequest("GET", "https://some-host:8080", nil) + require.Nil(t, err) + + proxyURL, err := ProxyFromClient(c)(req) + assert.Nil(t, proxyURL) + assert.Nil(t, err) +}