44cfe95394
If a redirect occurs, we don't close the HTTP body, forcing us to leave the current connection open and causing us to eventually run out of file handles. Ensure we close the body if we're not using it so that we can reuse the connection, which is both more efficient and nicer to the remote server.
549 lines
13 KiB
Go
549 lines
13 KiB
Go
package lfshttp
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/textproto"
|
|
"net/url"
|
|
"os"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/git-lfs/git-lfs/config"
|
|
"github.com/git-lfs/git-lfs/creds"
|
|
"github.com/git-lfs/git-lfs/errors"
|
|
"github.com/git-lfs/git-lfs/tools"
|
|
"github.com/rubyist/tracerx"
|
|
)
|
|
|
|
const MediaType = "application/vnd.git-lfs+json; charset=utf-8"
|
|
|
|
var (
|
|
UserAgent = "git-lfs"
|
|
httpRE = regexp.MustCompile(`\Ahttps?://`)
|
|
)
|
|
|
|
var hintFileUrl = strings.TrimSpace(`
|
|
hint: The remote resolves to a file:// URL, which can only work with a
|
|
hint: standalone transfer agent. See section "Using a Custom Transfer Type
|
|
hint: without the API server" in custom-transfers.md for details.
|
|
`)
|
|
|
|
type Client struct {
|
|
SSH SSHResolver
|
|
|
|
DialTimeout int
|
|
KeepaliveTimeout int
|
|
TLSTimeout int
|
|
ConcurrentTransfers int
|
|
SkipSSLVerify bool
|
|
|
|
Verbose bool
|
|
DebuggingVerbose bool
|
|
VerboseOut io.Writer
|
|
|
|
hostClients map[string]*http.Client
|
|
clientMu sync.Mutex
|
|
|
|
httpLogger *syncLogger
|
|
|
|
gitEnv config.Environment
|
|
osEnv config.Environment
|
|
uc *config.URLConfig
|
|
|
|
credHelperContext *creds.CredentialHelperContext
|
|
|
|
sshTries int
|
|
}
|
|
|
|
func NewClient(ctx Context) (*Client, error) {
|
|
if ctx == nil {
|
|
ctx = NewContext(nil, nil, nil)
|
|
}
|
|
|
|
gitEnv := ctx.GitEnv()
|
|
osEnv := ctx.OSEnv()
|
|
|
|
cacheCreds := gitEnv.Bool("lfs.cachecredentials", true)
|
|
var sshResolver SSHResolver = &sshAuthClient{os: osEnv, git: gitEnv}
|
|
if cacheCreds {
|
|
sshResolver = withSSHCache(sshResolver)
|
|
}
|
|
|
|
c := &Client{
|
|
SSH: sshResolver,
|
|
DialTimeout: gitEnv.Int("lfs.dialtimeout", 0),
|
|
KeepaliveTimeout: gitEnv.Int("lfs.keepalive", 0),
|
|
TLSTimeout: gitEnv.Int("lfs.tlstimeout", 0),
|
|
ConcurrentTransfers: gitEnv.Int("lfs.concurrenttransfers", 3),
|
|
SkipSSLVerify: !gitEnv.Bool("http.sslverify", true) || osEnv.Bool("GIT_SSL_NO_VERIFY", false),
|
|
Verbose: osEnv.Bool("GIT_CURL_VERBOSE", false),
|
|
DebuggingVerbose: osEnv.Bool("LFS_DEBUG_HTTP", false),
|
|
gitEnv: gitEnv,
|
|
osEnv: osEnv,
|
|
uc: config.NewURLConfig(gitEnv),
|
|
sshTries: gitEnv.Int("lfs.ssh.retries", 5),
|
|
credHelperContext: creds.NewCredentialHelperContext(gitEnv, osEnv),
|
|
}
|
|
|
|
return c, nil
|
|
}
|
|
|
|
func (c *Client) GitEnv() config.Environment {
|
|
return c.gitEnv
|
|
}
|
|
|
|
func (c *Client) OSEnv() config.Environment {
|
|
return c.osEnv
|
|
}
|
|
|
|
func (c *Client) URLConfig() *config.URLConfig {
|
|
return c.uc
|
|
}
|
|
|
|
func (c *Client) NewRequest(method string, e Endpoint, suffix string, body interface{}) (*http.Request, error) {
|
|
if strings.HasPrefix(e.Url, "file://") {
|
|
// Initial `\n` to avoid overprinting `Downloading LFS...`.
|
|
fmt.Fprintf(os.Stderr, "\n%s\n", hintFileUrl)
|
|
}
|
|
|
|
sshRes, err := c.sshResolveWithRetries(e, method)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
prefix := e.Url
|
|
if len(sshRes.Href) > 0 {
|
|
prefix = sshRes.Href
|
|
}
|
|
|
|
if !httpRE.MatchString(prefix) {
|
|
urlfragment := strings.SplitN(prefix, "?", 2)[0]
|
|
return nil, fmt.Errorf("missing protocol: %q", urlfragment)
|
|
}
|
|
|
|
req, err := http.NewRequest(method, joinURL(prefix, suffix), nil)
|
|
if err != nil {
|
|
return req, err
|
|
}
|
|
|
|
for key, value := range sshRes.Header {
|
|
req.Header.Set(key, value)
|
|
}
|
|
req.Header.Set("Accept", MediaType)
|
|
|
|
if body != nil {
|
|
if merr := MarshalToRequest(req, body); merr != nil {
|
|
return req, merr
|
|
}
|
|
req.Header.Set("Content-Type", MediaType)
|
|
}
|
|
|
|
return req, err
|
|
}
|
|
|
|
const slash = "/"
|
|
|
|
func joinURL(prefix, suffix string) string {
|
|
if strings.HasSuffix(prefix, slash) {
|
|
return prefix + suffix
|
|
}
|
|
return prefix + slash + suffix
|
|
}
|
|
|
|
// Do sends an HTTP request to get an HTTP response. It wraps net/http, adding
|
|
// extra headers, redirection handling, and error reporting.
|
|
func (c *Client) Do(req *http.Request) (*http.Response, error) {
|
|
req.Header = c.ExtraHeadersFor(req)
|
|
|
|
return c.do(req, "", nil)
|
|
}
|
|
|
|
// do performs an *http.Request respecting redirects, and handles the response
|
|
// as defined in c.handleResponse. Notably, it does not alter the headers for
|
|
// the request argument in any way.
|
|
func (c *Client) do(req *http.Request, remote string, via []*http.Request) (*http.Response, error) {
|
|
req.Header.Set("User-Agent", UserAgent)
|
|
|
|
res, err := c.doWithRedirects(c.HttpClient(req.Host), req, remote, via)
|
|
if err != nil {
|
|
return res, err
|
|
}
|
|
|
|
return res, c.handleResponse(res)
|
|
}
|
|
|
|
// Close closes any resources that this client opened.
|
|
func (c *Client) Close() error {
|
|
return c.httpLogger.Close()
|
|
}
|
|
|
|
func (c *Client) sshResolveWithRetries(e Endpoint, method string) (*sshAuthResponse, error) {
|
|
var sshRes sshAuthResponse
|
|
var err error
|
|
|
|
requests := tools.MaxInt(0, c.sshTries) + 1
|
|
for i := 0; i < requests; i++ {
|
|
sshRes, err = c.SSH.Resolve(e, method)
|
|
if err == nil {
|
|
return &sshRes, nil
|
|
}
|
|
|
|
tracerx.Printf(
|
|
"ssh: %s failed, error: %s, message: %s (try: %d/%d)",
|
|
e.SshUserAndHost, err.Error(), sshRes.Message, i,
|
|
requests,
|
|
)
|
|
}
|
|
|
|
if len(sshRes.Message) > 0 {
|
|
return nil, errors.Wrap(err, sshRes.Message)
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
func (c *Client) ExtraHeadersFor(req *http.Request) http.Header {
|
|
extraHeaders := c.extraHeaders(req.URL)
|
|
if len(extraHeaders) == 0 {
|
|
return req.Header
|
|
}
|
|
|
|
copy := make(http.Header, len(req.Header))
|
|
for k, vs := range req.Header {
|
|
copy[k] = vs
|
|
}
|
|
|
|
for k, vs := range extraHeaders {
|
|
for _, v := range vs {
|
|
copy[k] = append(copy[k], v)
|
|
}
|
|
}
|
|
return copy
|
|
}
|
|
|
|
func (c *Client) extraHeaders(u *url.URL) map[string][]string {
|
|
hdrs := c.uc.GetAll("http", u.String(), "extraHeader")
|
|
m := make(map[string][]string, len(hdrs))
|
|
|
|
for _, hdr := range hdrs {
|
|
parts := strings.SplitN(hdr, ":", 2)
|
|
if len(parts) < 2 {
|
|
continue
|
|
}
|
|
|
|
k, v := parts[0], strings.TrimSpace(parts[1])
|
|
// If header keys are given in non-canonicalized form (e.g.,
|
|
// "AUTHORIZATION" as opposed to "Authorization") they will not
|
|
// be returned in calls to net/http.Header.Get().
|
|
//
|
|
// So, we avoid this problem by first canonicalizing header keys
|
|
// for extra headers.
|
|
k = textproto.CanonicalMIMEHeaderKey(k)
|
|
|
|
m[k] = append(m[k], v)
|
|
}
|
|
return m
|
|
}
|
|
|
|
func (c *Client) DoWithRedirect(cli *http.Client, req *http.Request, remote string, via []*http.Request) (*http.Request, *http.Response, error) {
|
|
tracedReq, err := c.traceRequest(req)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
var retries int
|
|
if n, ok := Retries(req); ok {
|
|
retries = n
|
|
} else {
|
|
retries = defaultRequestRetries
|
|
}
|
|
|
|
var res *http.Response
|
|
|
|
requests := tools.MaxInt(0, retries) + 1
|
|
for i := 0; i < requests; i++ {
|
|
res, err = cli.Do(req)
|
|
if err == nil {
|
|
break
|
|
}
|
|
|
|
if seek, ok := req.Body.(io.Seeker); ok {
|
|
seek.Seek(0, io.SeekStart)
|
|
}
|
|
|
|
c.traceResponse(req, tracedReq, nil)
|
|
}
|
|
|
|
if err != nil {
|
|
c.traceResponse(req, tracedReq, nil)
|
|
return nil, nil, err
|
|
}
|
|
|
|
if res == nil {
|
|
return nil, nil, nil
|
|
}
|
|
|
|
c.traceResponse(req, tracedReq, res)
|
|
|
|
if res.StatusCode != 301 &&
|
|
res.StatusCode != 302 &&
|
|
res.StatusCode != 303 &&
|
|
res.StatusCode != 307 &&
|
|
res.StatusCode != 308 {
|
|
|
|
// Above are the list of 3xx status codes that we know
|
|
// how to handle below. If the status code contained in
|
|
// the HTTP response was none of them, return the (res,
|
|
// err) tuple as-is, otherwise handle the redirect.
|
|
return nil, res, c.handleResponse(res)
|
|
}
|
|
|
|
redirectTo := res.Header.Get("Location")
|
|
locurl, err := url.Parse(redirectTo)
|
|
if err == nil && !locurl.IsAbs() {
|
|
locurl = req.URL.ResolveReference(locurl)
|
|
redirectTo = locurl.String()
|
|
}
|
|
|
|
via = append(via, req)
|
|
if len(via) >= 3 {
|
|
return nil, res, errors.New("too many redirects")
|
|
}
|
|
|
|
redirectedReq, err := newRequestForRetry(req, redirectTo)
|
|
if err != nil {
|
|
return nil, res, err
|
|
}
|
|
|
|
res.Body.Close()
|
|
|
|
return redirectedReq, nil, nil
|
|
}
|
|
|
|
func (c *Client) doWithRedirects(cli *http.Client, req *http.Request, remote string, via []*http.Request) (*http.Response, error) {
|
|
redirectedReq, res, err := c.DoWithRedirect(cli, req, remote, via)
|
|
if err != nil || res != nil {
|
|
return res, err
|
|
}
|
|
|
|
if redirectedReq == nil {
|
|
return nil, errors.New("failed to redirect request")
|
|
}
|
|
|
|
return c.doWithRedirects(cli, redirectedReq, remote, via)
|
|
}
|
|
|
|
func (c *Client) HttpClient(host string) *http.Client {
|
|
c.clientMu.Lock()
|
|
defer c.clientMu.Unlock()
|
|
|
|
if c.gitEnv == nil {
|
|
c.gitEnv = make(testEnv)
|
|
}
|
|
|
|
if c.osEnv == nil {
|
|
c.osEnv = make(testEnv)
|
|
}
|
|
|
|
if c.hostClients == nil {
|
|
c.hostClients = make(map[string]*http.Client)
|
|
}
|
|
|
|
if client, ok := c.hostClients[host]; ok {
|
|
return client
|
|
}
|
|
|
|
concurrentTransfers := c.ConcurrentTransfers
|
|
if concurrentTransfers < 1 {
|
|
concurrentTransfers = 8
|
|
}
|
|
|
|
dialtime := c.DialTimeout
|
|
if dialtime < 1 {
|
|
dialtime = 30
|
|
}
|
|
|
|
keepalivetime := c.KeepaliveTimeout
|
|
if keepalivetime < 1 {
|
|
keepalivetime = 1800
|
|
}
|
|
|
|
tlstime := c.TLSTimeout
|
|
if tlstime < 1 {
|
|
tlstime = 30
|
|
}
|
|
|
|
tr := &http.Transport{
|
|
Proxy: proxyFromClient(c),
|
|
TLSHandshakeTimeout: time.Duration(tlstime) * time.Second,
|
|
MaxIdleConnsPerHost: concurrentTransfers,
|
|
}
|
|
|
|
activityTimeout := 30
|
|
if v, ok := c.uc.Get("lfs", fmt.Sprintf("https://%v", host), "activitytimeout"); ok {
|
|
if i, err := strconv.Atoi(v); err == nil {
|
|
activityTimeout = i
|
|
} else {
|
|
activityTimeout = 0
|
|
}
|
|
}
|
|
|
|
dialer := &net.Dialer{
|
|
Timeout: time.Duration(dialtime) * time.Second,
|
|
KeepAlive: time.Duration(keepalivetime) * time.Second,
|
|
DualStack: true,
|
|
}
|
|
|
|
if activityTimeout > 0 {
|
|
activityDuration := time.Duration(activityTimeout) * time.Second
|
|
tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
c, err := dialer.DialContext(ctx, network, addr)
|
|
if c == nil {
|
|
return c, err
|
|
}
|
|
if tc, ok := c.(*net.TCPConn); ok {
|
|
tc.SetKeepAlive(true)
|
|
tc.SetKeepAlivePeriod(dialer.KeepAlive)
|
|
}
|
|
return &deadlineConn{Timeout: activityDuration, Conn: c}, err
|
|
}
|
|
} else {
|
|
tr.DialContext = dialer.DialContext
|
|
}
|
|
|
|
tr.TLSClientConfig = &tls.Config{}
|
|
|
|
if isClientCertEnabledForHost(c, host) {
|
|
tracerx.Printf("http: client cert for %s", host)
|
|
cert := getClientCertForHost(c, host)
|
|
if cert != nil {
|
|
tr.TLSClientConfig.Certificates = []tls.Certificate{*cert}
|
|
tr.TLSClientConfig.BuildNameToCertificate()
|
|
}
|
|
}
|
|
|
|
if isCertVerificationDisabledForHost(c, host) {
|
|
tr.TLSClientConfig.InsecureSkipVerify = true
|
|
} else {
|
|
tr.TLSClientConfig.RootCAs = getRootCAsForHost(c, host)
|
|
}
|
|
|
|
httpClient := &http.Client{
|
|
Transport: tr,
|
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
|
return http.ErrUseLastResponse
|
|
},
|
|
}
|
|
|
|
c.hostClients[host] = httpClient
|
|
if c.VerboseOut == nil {
|
|
c.VerboseOut = os.Stderr
|
|
}
|
|
|
|
return httpClient
|
|
}
|
|
|
|
func (c *Client) CurrentUser() (string, string) {
|
|
userName, _ := c.gitEnv.Get("user.name")
|
|
userEmail, _ := c.gitEnv.Get("user.email")
|
|
return userName, userEmail
|
|
}
|
|
|
|
func newRequestForRetry(req *http.Request, location string) (*http.Request, error) {
|
|
newReq, err := http.NewRequest(req.Method, location, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if req.URL.Scheme == "https" && newReq.URL.Scheme == "http" {
|
|
return nil, errors.New("lfsapi/client: refusing insecure redirect, https->http")
|
|
}
|
|
|
|
sameHost := req.URL.Host == newReq.URL.Host
|
|
for key := range req.Header {
|
|
if key == "Authorization" {
|
|
if !sameHost {
|
|
continue
|
|
}
|
|
}
|
|
newReq.Header.Set(key, req.Header.Get(key))
|
|
}
|
|
|
|
oldestURL := strings.SplitN(req.URL.String(), "?", 2)[0]
|
|
newURL := strings.SplitN(newReq.URL.String(), "?", 2)[0]
|
|
tracerx.Printf("api: redirect %s %s to %s", req.Method, oldestURL, newURL)
|
|
|
|
// This body will have already been rewound from a call to
|
|
// lfsapi.Client.traceRequest().
|
|
newReq.Body = req.Body
|
|
newReq.ContentLength = req.ContentLength
|
|
|
|
// Copy the request's context.Context, if any.
|
|
newReq = newReq.WithContext(req.Context())
|
|
|
|
return newReq, nil
|
|
}
|
|
|
|
type deadlineConn struct {
|
|
Timeout time.Duration
|
|
net.Conn
|
|
}
|
|
|
|
func (c *deadlineConn) Read(b []byte) (int, error) {
|
|
if err := c.Conn.SetDeadline(time.Now().Add(c.Timeout)); err != nil {
|
|
return 0, err
|
|
}
|
|
return c.Conn.Read(b)
|
|
}
|
|
|
|
func (c *deadlineConn) Write(b []byte) (int, error) {
|
|
if err := c.Conn.SetDeadline(time.Now().Add(c.Timeout)); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
return c.Conn.Write(b)
|
|
}
|
|
|
|
func init() {
|
|
UserAgent = config.VersionDesc
|
|
}
|
|
|
|
type testEnv map[string]string
|
|
|
|
func (e testEnv) Get(key string) (v string, ok bool) {
|
|
v, ok = e[key]
|
|
return
|
|
}
|
|
|
|
func (e testEnv) GetAll(key string) []string {
|
|
if v, ok := e.Get(key); ok {
|
|
return []string{v}
|
|
}
|
|
return make([]string, 0)
|
|
}
|
|
|
|
func (e testEnv) Int(key string, def int) int {
|
|
s, _ := e.Get(key)
|
|
return config.Int(s, def)
|
|
}
|
|
|
|
func (e testEnv) Bool(key string, def bool) bool {
|
|
s, _ := e.Get(key)
|
|
return config.Bool(s, def)
|
|
}
|
|
|
|
func (e testEnv) All() map[string][]string {
|
|
m := make(map[string][]string)
|
|
for k, _ := range e {
|
|
m[k] = e.GetAll(k)
|
|
}
|
|
return m
|
|
}
|