lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lfsapi: add proxy support

Browse Source
This commit is contained in:
risk danger olson 2016-12-20 10:29:26 -07:00
parent 0d6f87302e
commit 1ba67ca09c
5 changed files with 303 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lfsapi/auth_test.go
View File

@ -63,7 +63,7 @@ func TestDoWithAuthApprove(t *testing.T) {
creds := newMockCredentialHelper()
c := &Client{
Credentials: creds,
Endpoints: NewEndpointFinder(gitEnv(map[string]string{
Endpoints: NewEndpointFinder(testEnv(map[string]string{
"lfs.url": srv.URL,
})),
}
@ -125,7 +125,7 @@ func TestDoWithAuthReject(t *testing.T) {
c := &Client{
Credentials: creds,
Endpoints: NewEndpointFinder(gitEnv(map[string]string{
Endpoints: NewEndpointFinder(testEnv(map[string]string{
"lfs.url": srv.URL,
})),
}
@ -343,7 +343,7 @@ func TestGetCredentials(t *testing.T) {
for _, check := range checks {
t.Logf("Checking %q", check.Desc)
ef := NewEndpointFinder(gitEnv(check.Config))
ef := NewEndpointFinder(testEnv(check.Config))
req, err := http.NewRequest(check.Method, check.Href, nil)
if err != nil {

78
lfsapi/endpoint_finder_test.go
View File

@ -1,14 +1,13 @@
package lfsapi
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestEndpointDefaultsToOrigin(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.lfsurl": "abc",
}))
@ -19,7 +18,7 @@ func TestEndpointDefaultsToOrigin(t *testing.T) {
}
func TestEndpointOverridesOrigin(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"lfs.url": "abc",
"remote.origin.lfsurl": "def",
}))
@ -31,7 +30,7 @@ func TestEndpointOverridesOrigin(t *testing.T) {
}
func TestEndpointNoOverrideDefaultRemote(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.lfsurl": "abc",
"remote.other.lfsurl": "def",
}))
@ -43,7 +42,7 @@ func TestEndpointNoOverrideDefaultRemote(t *testing.T) {
}
func TestEndpointUseAlternateRemote(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.lfsurl": "abc",
"remote.other.lfsurl": "def",
}))
@ -55,7 +54,7 @@ func TestEndpointUseAlternateRemote(t *testing.T) {
}
func TestEndpointAddsLfsSuffix(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "https://example.com/foo/bar",
}))
@ -66,7 +65,7 @@ func TestEndpointAddsLfsSuffix(t *testing.T) {
}
func TestBareEndpointAddsLfsSuffix(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "https://example.com/foo/bar.git",
}))
@ -77,7 +76,7 @@ func TestBareEndpointAddsLfsSuffix(t *testing.T) {
}
func TestEndpointSeparateClonePushUrl(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "https://example.com/foo/bar.git",
"remote.origin.pushurl": "https://readwrite.com/foo/bar.git",
}))
@ -94,7 +93,7 @@ func TestEndpointSeparateClonePushUrl(t *testing.T) {
}
func TestEndpointOverriddenSeparateClonePushLfsUrl(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "https://example.com/foo/bar.git",
"remote.origin.pushurl": "https://readwrite.com/foo/bar.git",
"remote.origin.lfsurl": "https://examplelfs.com/foo/bar",
@ -113,7 +112,7 @@ func TestEndpointOverriddenSeparateClonePushLfsUrl(t *testing.T) {
}
func TestEndpointGlobalSeparateLfsPush(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"lfs.url": "https://readonly.com/foo/bar",
"lfs.pushurl": "https://write.com/foo/bar",
}))
@ -130,7 +129,7 @@ func TestEndpointGlobalSeparateLfsPush(t *testing.T) {
}
func TestSSHEndpointOverridden(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "git@example.com:foo/bar",
"remote.origin.lfsurl": "lfs",
}))
@ -143,7 +142,7 @@ func TestSSHEndpointOverridden(t *testing.T) {
}
func TestSSHEndpointAddsLfsSuffix(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "ssh://git@example.com/foo/bar",
}))
@ -155,7 +154,7 @@ func TestSSHEndpointAddsLfsSuffix(t *testing.T) {
}
func TestSSHCustomPortEndpointAddsLfsSuffix(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "ssh://git@example.com:9000/foo/bar",
}))
@ -167,7 +166,7 @@ func TestSSHCustomPortEndpointAddsLfsSuffix(t *testing.T) {
}
func TestBareSSHEndpointAddsLfsSuffix(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "git@example.com:foo/bar.git",
}))
@ -179,7 +178,7 @@ func TestBareSSHEndpointAddsLfsSuffix(t *testing.T) {
}
func TestSSHEndpointFromGlobalLfsUrl(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"lfs.url": "git@example.com:foo/bar.git",
}))
@ -191,7 +190,7 @@ func TestSSHEndpointFromGlobalLfsUrl(t *testing.T) {
}
func TestHTTPEndpointAddsLfsSuffix(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "http://example.com/foo/bar",
}))
@ -203,7 +202,7 @@ func TestHTTPEndpointAddsLfsSuffix(t *testing.T) {
}
func TestBareHTTPEndpointAddsLfsSuffix(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "http://example.com/foo/bar.git",
}))
@ -215,7 +214,7 @@ func TestBareHTTPEndpointAddsLfsSuffix(t *testing.T) {
}
func TestGitEndpointAddsLfsSuffix(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "git://example.com/foo/bar",
}))
@ -227,7 +226,7 @@ func TestGitEndpointAddsLfsSuffix(t *testing.T) {
}
func TestGitEndpointAddsLfsSuffixWithCustomProtocol(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "git://example.com/foo/bar",
"lfs.gitprotocol": "http",
}))
@ -240,7 +239,7 @@ func TestGitEndpointAddsLfsSuffixWithCustomProtocol(t *testing.T) {
}
func TestBareGitEndpointAddsLfsSuffix(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"remote.origin.url": "git://example.com/foo/bar.git",
}))
@ -267,7 +266,7 @@ func TestAccessConfig(t *testing.T) {
}
for value, expected := range tests {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"lfs.url": "http://example.com",
"lfs.http://example.com.access": value,
"lfs.https://example.com.access": "bad",
@ -286,7 +285,7 @@ func TestAccessConfig(t *testing.T) {
// Test again but with separate push url
for value, expected := range tests {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"lfs.url": "http://example.com",
"lfs.pushurl": "http://examplepush.com",
"lfs.http://example.com.access": value,
@ -313,7 +312,7 @@ func TestAccessAbsentConfig(t *testing.T) {
}
func TestSetAccess(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{}))
finder := NewEndpointFinder(testEnv(map[string]string{}))
assert.Equal(t, NoneAccess, finder.AccessFor("http://example.com"))
finder.SetAccess("http://example.com", NTLMAccess)
@ -321,7 +320,7 @@ func TestSetAccess(t *testing.T) {
}
func TestChangeAccess(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"lfs.http://example.com.access": "basic",
}))
@ -331,7 +330,7 @@ func TestChangeAccess(t *testing.T) {
}
func TestDeleteAccessWithNone(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"lfs.http://example.com.access": "basic",
}))
@ -341,7 +340,7 @@ func TestDeleteAccessWithNone(t *testing.T) {
}
func TestDeleteAccessWithEmptyString(t *testing.T) {
finder := NewEndpointFinder(gitEnv(map[string]string{
finder := NewEndpointFinder(testEnv(map[string]string{
"lfs.http://example.com.access": "basic",
}))
@ -349,30 +348,3 @@ func TestDeleteAccessWithEmptyString(t *testing.T) {
finder.SetAccess("http://example.com", Access(""))
assert.Equal(t, NoneAccess, finder.AccessFor("http://example.com"))
}
type gitEnv map[string]string
func (e gitEnv) Get(key string) (string, bool) {
v, ok := e[key]
return v, ok
}
func (e gitEnv) Bool(key string, def bool) (val bool) {
s, _ := e.Get(key)
if len(s) == 0 {
return def
}
switch strings.ToLower(s) {
case "true", "1", "on", "yes", "t":
return true
case "false", "0", "off", "no", "f":
return false
default:
return false
}
}
func (e gitEnv) All() map[string]string {
return e
}

81
lfsapi/lfsapi.go
View File

@ -5,6 +5,7 @@ import (
"net"
"net/http"
"regexp"
"strings"
"sync"
"time"
@ -26,22 +27,39 @@ type Client struct {
TLSTimeout int `git:"lfs.tlstimeout"`
ConcurrentTransfers int `git:"lfs.concurrenttransfers"`
HTTPSProxy string
HTTPProxy string
NoProxy string
hostClients map[string]*http.Client
clientMu sync.Mutex
}
func NewClient(osEnv env, gitEnv env) (*Client, error) {
if osEnv == nil {
osEnv = make(testEnv)
}
if gitEnv == nil {
gitEnv = make(testEnv)
}
netrc, err := ParseNetrc(osEnv)
if err != nil {
return nil, err
}
httpsProxy, httpProxy, noProxy := getProxyServers(osEnv, gitEnv)
return &Client{
Endpoints: NewEndpointFinder(gitEnv),
Credentials: &CommandCredentialHelper{
SkipPrompt: !osEnv.Bool("GIT_TERMINAL_PROMPT", true),
},
Netrc: netrc,
Netrc: netrc,
HTTPSProxy: httpsProxy,
HTTPProxy: httpProxy,
NoProxy: noProxy,
}, nil
}
@ -87,6 +105,7 @@ func (c *Client) httpClient(host string) *http.Client {
}
tr := &http.Transport{
Proxy: ProxyFromClient(c),
Dial: (&net.Dialer{
Timeout: time.Duration(dialtime) * time.Second,
KeepAlive: time.Duration(keepalivetime) * time.Second,
@ -104,6 +123,37 @@ func (c *Client) httpClient(host string) *http.Client {
return httpClient
}
func getProxyServers(osEnv env, gitEnv env) (string, string, string) {
var httpsProxy string
httpProxy, _ := gitEnv.Get("http.proxy")
if strings.HasPrefix(httpProxy, "https://") {
httpsProxy = httpProxy
}
if len(httpsProxy) == 0 {
httpsProxy, _ = osEnv.Get("HTTPS_PROXY")
}
if len(httpsProxy) == 0 {
httpsProxy, _ = osEnv.Get("https_proxy")
}
if len(httpProxy) == 0 {
httpProxy, _ = osEnv.Get("HTTP_PROXY")
}
if len(httpProxy) == 0 {
httpProxy, _ = osEnv.Get("http_proxy")
}
noProxy, _ := osEnv.Get("NO_PROXY")
if len(noProxy) == 0 {
noProxy, _ = osEnv.Get("no_proxy")
}
return httpsProxy, httpProxy, noProxy
}
func decodeResponse(res *http.Response, obj interface{}) error {
ctype := res.Header.Get("Content-Type")
if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) {
@ -119,3 +169,32 @@ func decodeResponse(res *http.Response, obj interface{}) error {
return nil
}
// basic config.Environment implementation. Only used in tests, or as a zero
// value to NewClient().
type testEnv map[string]string
func (e testEnv) Get(key string) (string, bool) {
v, ok := e[key]
return v, ok
}
func (e testEnv) Bool(key string, def bool) (val bool) {
s, _ := e.Get(key)
if len(s) == 0 {
return def
}
switch strings.ToLower(s) {
case "true", "1", "on", "yes", "t":
return true
case "false", "0", "off", "no", "f":
return false
default:
return false
}
}
func (e testEnv) All() map[string]string {
return e
}

113
lfsapi/proxy.go Normal file
View File

@ -0,0 +1,113 @@
package lfsapi
import (
"net/http"
"net/url"
"strings"
"fmt"
)
// Logic is copied, with small changes, from "net/http".ProxyFromEnvironment in the go std lib.
func ProxyFromClient(c *Client) func(req *http.Request) (*url.URL, error) {
httpProxy := c.HTTPProxy
httpsProxy := c.HTTPSProxy
noProxy := c.NoProxy
return func(req *http.Request) (*url.URL, error) {
var proxy string
if req.URL.Scheme == "https" {
proxy = httpsProxy
}
if len(proxy) == 0 {
proxy = httpProxy
}
if len(proxy) == 0 {
return nil, nil
}
if !useProxy(noProxy, canonicalAddr(req.URL)) {
return nil, nil
}
proxyURL, err := url.Parse(proxy)
if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") {
// proxy was bogus. Try prepending "http://" to it and
// see if that parses correctly. If not, we fall
// through and complain about the original one.
if httpProxyURL, httpErr := url.Parse("http://" + proxy); httpErr == nil {
return httpProxyURL, nil
}
}
if err != nil {
return nil, fmt.Errorf("invalid proxy address: %q: %v", proxy, err)
}
return proxyURL, nil
}
}
// canonicalAddr returns url.Host but always with a ":port" suffix
// Copied from "net/http".ProxyFromEnvironment in the go std lib.
func canonicalAddr(url *url.URL) string {
addr := url.Host
if !hasPort(addr) {
return addr + ":" + portMap[url.Scheme]
}
return addr
}
// useProxy reports whether requests to addr should use a proxy,
// according to the noProxy or noProxy environment variable.
// addr is always a canonicalAddr with a host and port.
// Copied from "net/http".ProxyFromEnvironment in the go std lib
// and adapted to allow proxy usage even for localhost.
func useProxy(noProxy, addr string) bool {
if len(addr) == 0 {
return true
}
if noProxy == "*" {
return false
}
addr = strings.ToLower(strings.TrimSpace(addr))
if hasPort(addr) {
addr = addr[:strings.LastIndex(addr, ":")]
}
for _, p := range strings.Split(noProxy, ",") {
p = strings.ToLower(strings.TrimSpace(p))
if len(p) == 0 {
continue
}
if hasPort(p) {
p = p[:strings.LastIndex(p, ":")]
}
if addr == p {
return false
}
if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) {
// noProxy ".foo.com" matches "bar.foo.com" or "foo.com"
return false
}
if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' {
// noProxy "foo.com" matches "bar.foo.com"
return false
}
}
return true
}
// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
// return true if the string includes a port.
// Copied from "net/http".ProxyFromEnvironment in the go std lib.
func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
var (
portMap = map[string]string{
"http": "80",
"https": "443",
}
)

82
lfsapi/proxy_test.go Normal file
View File

@ -0,0 +1,82 @@
package lfsapi
import (
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestProxyFromGitConfig(t *testing.T) {
c, err := NewClient(testEnv(map[string]string{
"HTTPS_PROXY": "https://proxy-from-env:8080",
}), testEnv(map[string]string{
"http.proxy": "https://proxy-from-git-config:8080",
}))
require.Nil(t, err)
req, err := http.NewRequest("GET", "https://some-host.com:123/foo/bar", nil)
require.Nil(t, err)
proxyURL, err := ProxyFromClient(c)(req)
assert.Equal(t, "proxy-from-git-config:8080", proxyURL.Host)
assert.Nil(t, err)
}
func TestHttpProxyFromGitConfig(t *testing.T) {
c, err := NewClient(testEnv(map[string]string{
"HTTPS_PROXY": "https://proxy-from-env:8080",
}), testEnv(map[string]string{
"http.proxy": "http://proxy-from-git-config:8080",
}))
require.Nil(t, err)
req, err := http.NewRequest("GET", "https://some-host.com:123/foo/bar", nil)
require.Nil(t, err)
proxyURL, err := ProxyFromClient(c)(req)
assert.Equal(t, "proxy-from-env:8080", proxyURL.Host)
assert.Nil(t, err)
}
func TestProxyFromEnvironment(t *testing.T) {
c, err := NewClient(testEnv(map[string]string{
"HTTPS_PROXY": "https://proxy-from-env:8080",
}), nil)
require.Nil(t, err)
req, err := http.NewRequest("GET", "https://some-host.com:123/foo/bar", nil)
require.Nil(t, err)
proxyURL, err := ProxyFromClient(c)(req)
assert.Equal(t, "proxy-from-env:8080", proxyURL.Host)
assert.Nil(t, err)
}
func TestProxyIsNil(t *testing.T) {
c := &Client{}
req, err := http.NewRequest("GET", "http://some-host.com:123/foo/bar", nil)
require.Nil(t, err)
proxyURL, err := ProxyFromClient(c)(req)
assert.Nil(t, proxyURL)
assert.Nil(t, err)
}
func TestProxyNoProxy(t *testing.T) {
c, err := NewClient(testEnv(map[string]string{
"NO_PROXY": "some-host",
}), testEnv(map[string]string{
"http.proxy": "https://proxy-from-git-config:8080",
}))
require.Nil(t, err)
req, err := http.NewRequest("GET", "https://some-host:8080", nil)
require.Nil(t, err)
proxyURL, err := ProxyFromClient(c)(req)
assert.Nil(t, proxyURL)
assert.Nil(t, err)
}