Merge pull request #3972 from bk2204/better-proxy-support
Improved proxy support
This commit is contained in:
commit
770719d44a
@ -1,12 +1,12 @@
|
||||
package lfshttp
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/git-lfs/git-lfs/config"
|
||||
"golang.org/x/net/http/httpproxy"
|
||||
)
|
||||
|
||||
// Logic is copied, with small changes, from "net/http".ProxyFromEnvironment in the go std lib.
|
||||
@ -27,23 +27,23 @@ func proxyFromClient(c *Client) func(req *http.Request) (*url.URL, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if !useProxy(noProxy, canonicalAddr(req.URL)) {
|
||||
return nil, nil
|
||||
cfg := &httpproxy.Config{
|
||||
HTTPProxy: proxy,
|
||||
HTTPSProxy: proxy,
|
||||
NoProxy: noProxy,
|
||||
CGI: false,
|
||||
}
|
||||
|
||||
proxyURL, err := url.Parse(proxy)
|
||||
if err != nil || !(strings.HasPrefix(proxyURL.Scheme, "http") || strings.HasPrefix(proxyURL.Scheme, "socks")) {
|
||||
// proxy was bogus. Try prepending "http://" to it and
|
||||
// see if that parses correctly. If not, we fall
|
||||
// through and complain about the original one.
|
||||
if httpProxyURL, httpErr := url.Parse("http://" + proxy); httpErr == nil {
|
||||
return httpProxyURL, nil
|
||||
}
|
||||
// We want to use the standard logic except that we want to
|
||||
// allow proxies for localhost, which the standard library does
|
||||
// not. Since the proxy code looks only at the URL, we
|
||||
// synthesize a fake URL except that we rewrite "localhost" to
|
||||
// "127.0.0.1" for purposes of looking up the proxy.
|
||||
u := *(req.URL)
|
||||
if u.Host == "localhost" {
|
||||
u.Host = "127.0.0.1"
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid proxy address: %q: %v", proxy, err)
|
||||
}
|
||||
return proxyURL, nil
|
||||
return cfg.ProxyFunc()(&u)
|
||||
}
|
||||
}
|
||||
|
||||
@ -85,67 +85,3 @@ func getProxyServers(u *url.URL, urlCfg *config.URLConfig, osEnv config.Environm
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// canonicalAddr returns url.Host but always with a ":port" suffix
|
||||
// Copied from "net/http".ProxyFromEnvironment in the go std lib.
|
||||
func canonicalAddr(url *url.URL) string {
|
||||
addr := url.Host
|
||||
if !hasPort(addr) {
|
||||
return addr + ":" + portMap[url.Scheme]
|
||||
}
|
||||
return addr
|
||||
}
|
||||
|
||||
// useProxy reports whether requests to addr should use a proxy,
|
||||
// according to the noProxy or noProxy environment variable.
|
||||
// addr is always a canonicalAddr with a host and port.
|
||||
// Copied from "net/http".ProxyFromEnvironment in the go std lib
|
||||
// and adapted to allow proxy usage even for localhost.
|
||||
func useProxy(noProxy, addr string) bool {
|
||||
if len(addr) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
if noProxy == "*" {
|
||||
return false
|
||||
}
|
||||
|
||||
addr = strings.ToLower(strings.TrimSpace(addr))
|
||||
if hasPort(addr) {
|
||||
addr = addr[:strings.LastIndex(addr, ":")]
|
||||
}
|
||||
|
||||
for _, p := range strings.Split(noProxy, ",") {
|
||||
p = strings.ToLower(strings.TrimSpace(p))
|
||||
if len(p) == 0 {
|
||||
continue
|
||||
}
|
||||
if hasPort(p) {
|
||||
p = p[:strings.LastIndex(p, ":")]
|
||||
}
|
||||
if addr == p {
|
||||
return false
|
||||
}
|
||||
if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) {
|
||||
// noProxy ".foo.com" matches "bar.foo.com" or "foo.com"
|
||||
return false
|
||||
}
|
||||
if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' {
|
||||
// noProxy "foo.com" matches "bar.foo.com"
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
|
||||
// return true if the string includes a port.
|
||||
// Copied from "net/http".ProxyFromEnvironment in the go std lib.
|
||||
func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
|
||||
|
||||
var (
|
||||
portMap = map[string]string{
|
||||
"http": "80",
|
||||
"https": "443",
|
||||
}
|
||||
)
|
||||
|
@ -96,6 +96,22 @@ func TestProxyNoProxy(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestProxyNoProxyWithWildcard(t *testing.T) {
|
||||
c, err := NewClient(NewContext(nil, map[string]string{
|
||||
"NO_PROXY": "*.example.com",
|
||||
}, map[string]string{
|
||||
"http.proxy": "https://proxy-from-git-config:8080",
|
||||
}))
|
||||
require.Nil(t, err)
|
||||
|
||||
req, err := http.NewRequest("GET", "https://foo.example.com:8080", nil)
|
||||
require.Nil(t, err)
|
||||
|
||||
proxyURL, err := proxyFromClient(c)(req)
|
||||
assert.Nil(t, proxyURL)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestSocksProxyFromEnvironment(t *testing.T) {
|
||||
c, err := NewClient(NewContext(nil, map[string]string{
|
||||
"HTTPS_PROXY": "socks5://proxy-from-env:3128",
|
||||
|
370
vendor/golang.org/x/net/http/httpproxy/proxy.go
generated
vendored
Normal file
370
vendor/golang.org/x/net/http/httpproxy/proxy.go
generated
vendored
Normal file
@ -0,0 +1,370 @@
|
||||
// Copyright 2017 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package httpproxy provides support for HTTP proxy determination
|
||||
// based on environment variables, as provided by net/http's
|
||||
// ProxyFromEnvironment function.
|
||||
//
|
||||
// The API is not subject to the Go 1 compatibility promise and may change at
|
||||
// any time.
|
||||
package httpproxy
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/net/idna"
|
||||
)
|
||||
|
||||
// Config holds configuration for HTTP proxy settings. See
|
||||
// FromEnvironment for details.
|
||||
type Config struct {
|
||||
// HTTPProxy represents the value of the HTTP_PROXY or
|
||||
// http_proxy environment variable. It will be used as the proxy
|
||||
// URL for HTTP requests and HTTPS requests unless overridden by
|
||||
// HTTPSProxy or NoProxy.
|
||||
HTTPProxy string
|
||||
|
||||
// HTTPSProxy represents the HTTPS_PROXY or https_proxy
|
||||
// environment variable. It will be used as the proxy URL for
|
||||
// HTTPS requests unless overridden by NoProxy.
|
||||
HTTPSProxy string
|
||||
|
||||
// NoProxy represents the NO_PROXY or no_proxy environment
|
||||
// variable. It specifies a string that contains comma-separated values
|
||||
// specifying hosts that should be excluded from proxying. Each value is
|
||||
// represented by an IP address prefix (1.2.3.4), an IP address prefix in
|
||||
// CIDR notation (1.2.3.4/8), a domain name, or a special DNS label (*).
|
||||
// An IP address prefix and domain name can also include a literal port
|
||||
// number (1.2.3.4:80).
|
||||
// A domain name matches that name and all subdomains. A domain name with
|
||||
// a leading "." matches subdomains only. For example "foo.com" matches
|
||||
// "foo.com" and "bar.foo.com"; ".y.com" matches "x.y.com" but not "y.com".
|
||||
// A single asterisk (*) indicates that no proxying should be done.
|
||||
// A best effort is made to parse the string and errors are
|
||||
// ignored.
|
||||
NoProxy string
|
||||
|
||||
// CGI holds whether the current process is running
|
||||
// as a CGI handler (FromEnvironment infers this from the
|
||||
// presence of a REQUEST_METHOD environment variable).
|
||||
// When this is set, ProxyForURL will return an error
|
||||
// when HTTPProxy applies, because a client could be
|
||||
// setting HTTP_PROXY maliciously. See https://golang.org/s/cgihttpproxy.
|
||||
CGI bool
|
||||
}
|
||||
|
||||
// config holds the parsed configuration for HTTP proxy settings.
|
||||
type config struct {
|
||||
// Config represents the original configuration as defined above.
|
||||
Config
|
||||
|
||||
// httpsProxy is the parsed URL of the HTTPSProxy if defined.
|
||||
httpsProxy *url.URL
|
||||
|
||||
// httpProxy is the parsed URL of the HTTPProxy if defined.
|
||||
httpProxy *url.URL
|
||||
|
||||
// ipMatchers represent all values in the NoProxy that are IP address
|
||||
// prefixes or an IP address in CIDR notation.
|
||||
ipMatchers []matcher
|
||||
|
||||
// domainMatchers represent all values in the NoProxy that are a domain
|
||||
// name or hostname & domain name
|
||||
domainMatchers []matcher
|
||||
}
|
||||
|
||||
// FromEnvironment returns a Config instance populated from the
|
||||
// environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the
|
||||
// lowercase versions thereof). HTTPS_PROXY takes precedence over
|
||||
// HTTP_PROXY for https requests.
|
||||
//
|
||||
// The environment values may be either a complete URL or a
|
||||
// "host[:port]", in which case the "http" scheme is assumed. An error
|
||||
// is returned if the value is a different form.
|
||||
func FromEnvironment() *Config {
|
||||
return &Config{
|
||||
HTTPProxy: getEnvAny("HTTP_PROXY", "http_proxy"),
|
||||
HTTPSProxy: getEnvAny("HTTPS_PROXY", "https_proxy"),
|
||||
NoProxy: getEnvAny("NO_PROXY", "no_proxy"),
|
||||
CGI: os.Getenv("REQUEST_METHOD") != "",
|
||||
}
|
||||
}
|
||||
|
||||
func getEnvAny(names ...string) string {
|
||||
for _, n := range names {
|
||||
if val := os.Getenv(n); val != "" {
|
||||
return val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ProxyFunc returns a function that determines the proxy URL to use for
|
||||
// a given request URL. Changing the contents of cfg will not affect
|
||||
// proxy functions created earlier.
|
||||
//
|
||||
// A nil URL and nil error are returned if no proxy is defined in the
|
||||
// environment, or a proxy should not be used for the given request, as
|
||||
// defined by NO_PROXY.
|
||||
//
|
||||
// As a special case, if req.URL.Host is "localhost" (with or without a
|
||||
// port number), then a nil URL and nil error will be returned.
|
||||
func (cfg *Config) ProxyFunc() func(reqURL *url.URL) (*url.URL, error) {
|
||||
// Preprocess the Config settings for more efficient evaluation.
|
||||
cfg1 := &config{
|
||||
Config: *cfg,
|
||||
}
|
||||
cfg1.init()
|
||||
return cfg1.proxyForURL
|
||||
}
|
||||
|
||||
func (cfg *config) proxyForURL(reqURL *url.URL) (*url.URL, error) {
|
||||
var proxy *url.URL
|
||||
if reqURL.Scheme == "https" {
|
||||
proxy = cfg.httpsProxy
|
||||
}
|
||||
if proxy == nil {
|
||||
proxy = cfg.httpProxy
|
||||
if proxy != nil && cfg.CGI {
|
||||
return nil, errors.New("refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy")
|
||||
}
|
||||
}
|
||||
if proxy == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if !cfg.useProxy(canonicalAddr(reqURL)) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return proxy, nil
|
||||
}
|
||||
|
||||
func parseProxy(proxy string) (*url.URL, error) {
|
||||
if proxy == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
proxyURL, err := url.Parse(proxy)
|
||||
if err != nil ||
|
||||
(proxyURL.Scheme != "http" &&
|
||||
proxyURL.Scheme != "https" &&
|
||||
proxyURL.Scheme != "socks5") {
|
||||
// proxy was bogus. Try prepending "http://" to it and
|
||||
// see if that parses correctly. If not, we fall
|
||||
// through and complain about the original one.
|
||||
if proxyURL, err := url.Parse("http://" + proxy); err == nil {
|
||||
return proxyURL, nil
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
|
||||
}
|
||||
return proxyURL, nil
|
||||
}
|
||||
|
||||
// useProxy reports whether requests to addr should use a proxy,
|
||||
// according to the NO_PROXY or no_proxy environment variable.
|
||||
// addr is always a canonicalAddr with a host and port.
|
||||
func (cfg *config) useProxy(addr string) bool {
|
||||
if len(addr) == 0 {
|
||||
return true
|
||||
}
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if host == "localhost" {
|
||||
return false
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip != nil {
|
||||
if ip.IsLoopback() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
addr = strings.ToLower(strings.TrimSpace(host))
|
||||
|
||||
if ip != nil {
|
||||
for _, m := range cfg.ipMatchers {
|
||||
if m.match(addr, port, ip) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range cfg.domainMatchers {
|
||||
if m.match(addr, port, ip) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *config) init() {
|
||||
if parsed, err := parseProxy(c.HTTPProxy); err == nil {
|
||||
c.httpProxy = parsed
|
||||
}
|
||||
if parsed, err := parseProxy(c.HTTPSProxy); err == nil {
|
||||
c.httpsProxy = parsed
|
||||
}
|
||||
|
||||
for _, p := range strings.Split(c.NoProxy, ",") {
|
||||
p = strings.ToLower(strings.TrimSpace(p))
|
||||
if len(p) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if p == "*" {
|
||||
c.ipMatchers = []matcher{allMatch{}}
|
||||
c.domainMatchers = []matcher{allMatch{}}
|
||||
return
|
||||
}
|
||||
|
||||
// IPv4/CIDR, IPv6/CIDR
|
||||
if _, pnet, err := net.ParseCIDR(p); err == nil {
|
||||
c.ipMatchers = append(c.ipMatchers, cidrMatch{cidr: pnet})
|
||||
continue
|
||||
}
|
||||
|
||||
// IPv4:port, [IPv6]:port
|
||||
phost, pport, err := net.SplitHostPort(p)
|
||||
if err == nil {
|
||||
if len(phost) == 0 {
|
||||
// There is no host part, likely the entry is malformed; ignore.
|
||||
continue
|
||||
}
|
||||
if phost[0] == '[' && phost[len(phost)-1] == ']' {
|
||||
phost = phost[1 : len(phost)-1]
|
||||
}
|
||||
} else {
|
||||
phost = p
|
||||
}
|
||||
// IPv4, IPv6
|
||||
if pip := net.ParseIP(phost); pip != nil {
|
||||
c.ipMatchers = append(c.ipMatchers, ipMatch{ip: pip, port: pport})
|
||||
continue
|
||||
}
|
||||
|
||||
if len(phost) == 0 {
|
||||
// There is no host part, likely the entry is malformed; ignore.
|
||||
continue
|
||||
}
|
||||
|
||||
// domain.com or domain.com:80
|
||||
// foo.com matches bar.foo.com
|
||||
// .domain.com or .domain.com:port
|
||||
// *.domain.com or *.domain.com:port
|
||||
if strings.HasPrefix(phost, "*.") {
|
||||
phost = phost[1:]
|
||||
}
|
||||
matchHost := false
|
||||
if phost[0] != '.' {
|
||||
matchHost = true
|
||||
phost = "." + phost
|
||||
}
|
||||
c.domainMatchers = append(c.domainMatchers, domainMatch{host: phost, port: pport, matchHost: matchHost})
|
||||
}
|
||||
}
|
||||
|
||||
var portMap = map[string]string{
|
||||
"http": "80",
|
||||
"https": "443",
|
||||
"socks5": "1080",
|
||||
}
|
||||
|
||||
// canonicalAddr returns url.Host but always with a ":port" suffix
|
||||
func canonicalAddr(url *url.URL) string {
|
||||
addr := url.Hostname()
|
||||
if v, err := idnaASCII(addr); err == nil {
|
||||
addr = v
|
||||
}
|
||||
port := url.Port()
|
||||
if port == "" {
|
||||
port = portMap[url.Scheme]
|
||||
}
|
||||
return net.JoinHostPort(addr, port)
|
||||
}
|
||||
|
||||
// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
|
||||
// return true if the string includes a port.
|
||||
func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
|
||||
|
||||
func idnaASCII(v string) (string, error) {
|
||||
// TODO: Consider removing this check after verifying performance is okay.
|
||||
// Right now punycode verification, length checks, context checks, and the
|
||||
// permissible character tests are all omitted. It also prevents the ToASCII
|
||||
// call from salvaging an invalid IDN, when possible. As a result it may be
|
||||
// possible to have two IDNs that appear identical to the user where the
|
||||
// ASCII-only version causes an error downstream whereas the non-ASCII
|
||||
// version does not.
|
||||
// Note that for correct ASCII IDNs ToASCII will only do considerably more
|
||||
// work, but it will not cause an allocation.
|
||||
if isASCII(v) {
|
||||
return v, nil
|
||||
}
|
||||
return idna.Lookup.ToASCII(v)
|
||||
}
|
||||
|
||||
func isASCII(s string) bool {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] >= utf8.RuneSelf {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// matcher represents the matching rule for a given value in the NO_PROXY list
|
||||
type matcher interface {
|
||||
// match returns true if the host and optional port or ip and optional port
|
||||
// are allowed
|
||||
match(host, port string, ip net.IP) bool
|
||||
}
|
||||
|
||||
// allMatch matches on all possible inputs
|
||||
type allMatch struct{}
|
||||
|
||||
func (a allMatch) match(host, port string, ip net.IP) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
type cidrMatch struct {
|
||||
cidr *net.IPNet
|
||||
}
|
||||
|
||||
func (m cidrMatch) match(host, port string, ip net.IP) bool {
|
||||
return m.cidr.Contains(ip)
|
||||
}
|
||||
|
||||
type ipMatch struct {
|
||||
ip net.IP
|
||||
port string
|
||||
}
|
||||
|
||||
func (m ipMatch) match(host, port string, ip net.IP) bool {
|
||||
if m.ip.Equal(ip) {
|
||||
return m.port == "" || m.port == port
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type domainMatch struct {
|
||||
host string
|
||||
port string
|
||||
|
||||
matchHost bool
|
||||
}
|
||||
|
||||
func (m domainMatch) match(host, port string, ip net.IP) bool {
|
||||
if strings.HasSuffix(host, m.host) || (m.matchHost && host == m.host[1:]) {
|
||||
return m.port == "" || m.port == port
|
||||
}
|
||||
return false
|
||||
}
|
1
vendor/modules.txt
vendored
1
vendor/modules.txt
vendored
@ -57,6 +57,7 @@ golang.org/x/crypto/md4
|
||||
golang.org/x/crypto/pbkdf2
|
||||
# golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a
|
||||
golang.org/x/net/http/httpguts
|
||||
golang.org/x/net/http/httpproxy
|
||||
golang.org/x/net/http2
|
||||
golang.org/x/net/http2/hpack
|
||||
golang.org/x/net/idna
|
||||
|
Loading…
Reference in New Issue
Block a user