2016-12-20 22:37:11 +00:00
|
|
|
package lfsapi
|
|
|
|
|
|
|
|
import (
|
2017-04-27 23:16:01 +00:00
|
|
|
"context"
|
2016-12-20 22:37:11 +00:00
|
|
|
"crypto/tls"
|
2017-04-27 23:16:01 +00:00
|
|
|
"fmt"
|
2017-08-24 21:51:08 +00:00
|
|
|
"io"
|
2016-12-20 22:37:11 +00:00
|
|
|
"net"
|
|
|
|
"net/http"
|
|
|
|
"net/url"
|
2016-12-22 18:01:51 +00:00
|
|
|
"os"
|
2017-04-27 23:16:01 +00:00
|
|
|
"strconv"
|
2016-12-20 22:37:11 +00:00
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
2017-04-14 20:11:44 +00:00
|
|
|
"github.com/git-lfs/git-lfs/config"
|
2016-12-20 22:37:11 +00:00
|
|
|
"github.com/git-lfs/git-lfs/errors"
|
2017-08-24 21:57:50 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tools"
|
2016-12-20 22:37:11 +00:00
|
|
|
"github.com/rubyist/tracerx"
|
|
|
|
)
|
|
|
|
|
2016-12-20 22:46:45 +00:00
|
|
|
var UserAgent = "git-lfs"
|
|
|
|
|
2017-01-04 17:10:56 +00:00
|
|
|
const MediaType = "application/vnd.git-lfs+json; charset=utf-8"
|
|
|
|
|
2016-12-22 21:35:57 +00:00
|
|
|
func (c *Client) NewRequest(method string, e Endpoint, suffix string, body interface{}) (*http.Request, error) {
|
2017-03-23 19:48:52 +00:00
|
|
|
sshRes, err := c.SSH.Resolve(e, method)
|
2017-01-04 23:23:46 +00:00
|
|
|
if err != nil {
|
|
|
|
tracerx.Printf("ssh: %s failed, error: %s, message: %s",
|
|
|
|
e.SshUserAndHost, err.Error(), sshRes.Message,
|
|
|
|
)
|
|
|
|
|
|
|
|
if len(sshRes.Message) > 0 {
|
|
|
|
return nil, errors.Wrap(err, sshRes.Message)
|
|
|
|
}
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
prefix := e.Url
|
|
|
|
if len(sshRes.Href) > 0 {
|
|
|
|
prefix = sshRes.Href
|
|
|
|
}
|
|
|
|
|
|
|
|
req, err := http.NewRequest(method, joinURL(prefix, suffix), nil)
|
2016-12-22 21:35:57 +00:00
|
|
|
if err != nil {
|
|
|
|
return req, err
|
|
|
|
}
|
|
|
|
|
2017-01-04 23:23:46 +00:00
|
|
|
for key, value := range sshRes.Header {
|
|
|
|
req.Header.Set(key, value)
|
|
|
|
}
|
2017-01-04 17:10:56 +00:00
|
|
|
req.Header.Set("Accept", MediaType)
|
|
|
|
|
2016-12-22 21:35:57 +00:00
|
|
|
if body != nil {
|
|
|
|
if merr := MarshalToRequest(req, body); merr != nil {
|
|
|
|
return req, merr
|
|
|
|
}
|
2017-01-04 17:10:56 +00:00
|
|
|
req.Header.Set("Content-Type", MediaType)
|
2016-12-22 21:35:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return req, err
|
|
|
|
}
|
|
|
|
|
|
|
|
const slash = "/"
|
|
|
|
|
|
|
|
func joinURL(prefix, suffix string) string {
|
|
|
|
if strings.HasSuffix(prefix, slash) {
|
|
|
|
return prefix + suffix
|
|
|
|
}
|
|
|
|
return prefix + slash + suffix
|
|
|
|
}
|
|
|
|
|
2016-12-20 22:37:11 +00:00
|
|
|
func (c *Client) Do(req *http.Request) (*http.Response, error) {
|
2017-04-17 17:28:55 +00:00
|
|
|
req.Header = c.extraHeadersFor(req)
|
2016-12-20 22:46:45 +00:00
|
|
|
req.Header.Set("User-Agent", UserAgent)
|
|
|
|
|
2016-12-20 22:37:11 +00:00
|
|
|
res, err := c.doWithRedirects(c.httpClient(req.Host), req, nil)
|
|
|
|
if err != nil {
|
|
|
|
return res, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return res, c.handleResponse(res)
|
|
|
|
}
|
|
|
|
|
2017-04-27 18:33:50 +00:00
|
|
|
// Close closes any resources that this client opened.
|
2017-04-27 17:00:27 +00:00
|
|
|
func (c *Client) Close() error {
|
2017-04-27 18:57:26 +00:00
|
|
|
return c.httpLogger.Close()
|
2017-04-27 17:00:27 +00:00
|
|
|
}
|
|
|
|
|
2017-04-17 17:28:55 +00:00
|
|
|
func (c *Client) extraHeadersFor(req *http.Request) http.Header {
|
2017-04-17 17:28:39 +00:00
|
|
|
copy := make(http.Header, len(req.Header))
|
2017-04-15 16:44:27 +00:00
|
|
|
for k, vs := range req.Header {
|
|
|
|
copy[k] = vs
|
|
|
|
}
|
|
|
|
|
|
|
|
for k, vs := range c.extraHeaders(req.URL) {
|
|
|
|
for _, v := range vs {
|
|
|
|
copy[k] = append(copy[k], v)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return copy
|
|
|
|
}
|
|
|
|
|
2017-04-15 16:44:08 +00:00
|
|
|
func (c *Client) extraHeaders(u *url.URL) map[string][]string {
|
|
|
|
hdrs := c.uc.GetAll("http", u.String(), "extraHeader")
|
|
|
|
m := make(map[string][]string, len(hdrs))
|
|
|
|
|
|
|
|
for _, hdr := range hdrs {
|
2017-04-17 17:21:50 +00:00
|
|
|
parts := strings.SplitN(hdr, ":", 2)
|
2017-04-15 16:44:08 +00:00
|
|
|
if len(parts) < 2 {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2017-04-17 17:21:50 +00:00
|
|
|
k, v := parts[0], strings.TrimSpace(parts[1])
|
2017-04-15 16:44:08 +00:00
|
|
|
|
|
|
|
m[k] = append(m[k], v)
|
|
|
|
}
|
|
|
|
return m
|
|
|
|
}
|
|
|
|
|
2016-12-20 22:37:11 +00:00
|
|
|
func (c *Client) doWithRedirects(cli *http.Client, req *http.Request, via []*http.Request) (*http.Response, error) {
|
2017-04-27 19:46:54 +00:00
|
|
|
tracedReq, err := c.traceRequest(req)
|
|
|
|
if err != nil {
|
2016-12-20 23:52:36 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-08-24 21:51:08 +00:00
|
|
|
var retries int
|
|
|
|
if n, ok := Retries(req); ok {
|
|
|
|
retries = n
|
|
|
|
} else {
|
|
|
|
retries = defaultRequestRetries
|
|
|
|
}
|
|
|
|
|
|
|
|
var res *http.Response
|
|
|
|
|
2017-09-07 15:04:48 +00:00
|
|
|
requests := tools.MaxInt(0, retries) + 1
|
|
|
|
for i := 0; i < requests; i++ {
|
2017-08-24 21:51:08 +00:00
|
|
|
res, err = cli.Do(req)
|
|
|
|
if err == nil {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
if seek, ok := req.Body.(io.Seeker); ok {
|
|
|
|
seek.Seek(0, io.SeekStart)
|
|
|
|
}
|
|
|
|
|
|
|
|
c.traceResponse(req, tracedReq, nil)
|
|
|
|
}
|
|
|
|
|
2016-12-20 22:37:11 +00:00
|
|
|
if err != nil {
|
2017-04-27 23:03:17 +00:00
|
|
|
c.traceResponse(req, tracedReq, nil)
|
2017-08-24 21:51:08 +00:00
|
|
|
return nil, err
|
2016-12-20 22:37:11 +00:00
|
|
|
}
|
2016-12-21 18:22:22 +00:00
|
|
|
|
2017-09-07 15:04:48 +00:00
|
|
|
if res == nil {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
2017-04-27 23:03:17 +00:00
|
|
|
c.traceResponse(req, tracedReq, res)
|
2016-12-20 22:37:11 +00:00
|
|
|
|
2017-05-18 21:04:59 +00:00
|
|
|
if res.StatusCode != 301 &&
|
|
|
|
res.StatusCode != 302 &&
|
|
|
|
res.StatusCode != 303 &&
|
|
|
|
res.StatusCode != 307 &&
|
|
|
|
res.StatusCode != 308 {
|
|
|
|
|
|
|
|
// Above are the list of 3xx status codes that we know
|
|
|
|
// how to handle below. If the status code contained in
|
|
|
|
// the HTTP response was none of them, return the (res,
|
|
|
|
// err) tuple as-is, otherwise handle the redirect.
|
2016-12-20 22:37:11 +00:00
|
|
|
return res, err
|
|
|
|
}
|
|
|
|
|
|
|
|
redirectTo := res.Header.Get("Location")
|
|
|
|
locurl, err := url.Parse(redirectTo)
|
|
|
|
if err == nil && !locurl.IsAbs() {
|
|
|
|
locurl = req.URL.ResolveReference(locurl)
|
|
|
|
redirectTo = locurl.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
via = append(via, req)
|
2016-12-21 17:06:55 +00:00
|
|
|
if len(via) >= 3 {
|
|
|
|
return res, errors.New("too many redirects")
|
2016-12-20 22:37:11 +00:00
|
|
|
}
|
|
|
|
|
2016-12-21 17:06:55 +00:00
|
|
|
redirectedReq, err := newRequestForRetry(req, redirectTo)
|
|
|
|
if err != nil {
|
2016-12-21 16:52:27 +00:00
|
|
|
return res, err
|
2016-12-20 22:37:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return c.doWithRedirects(cli, redirectedReq, via)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *Client) httpClient(host string) *http.Client {
|
|
|
|
c.clientMu.Lock()
|
|
|
|
defer c.clientMu.Unlock()
|
|
|
|
|
|
|
|
if c.gitEnv == nil {
|
2017-01-06 18:34:43 +00:00
|
|
|
c.gitEnv = make(TestEnv)
|
2016-12-20 22:37:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if c.osEnv == nil {
|
2017-01-06 18:34:43 +00:00
|
|
|
c.osEnv = make(TestEnv)
|
2016-12-20 22:37:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if c.hostClients == nil {
|
|
|
|
c.hostClients = make(map[string]*http.Client)
|
|
|
|
}
|
|
|
|
|
|
|
|
if client, ok := c.hostClients[host]; ok {
|
|
|
|
return client
|
|
|
|
}
|
|
|
|
|
|
|
|
concurrentTransfers := c.ConcurrentTransfers
|
|
|
|
if concurrentTransfers < 1 {
|
2017-08-12 04:23:07 +00:00
|
|
|
concurrentTransfers = 8
|
2016-12-20 22:37:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
dialtime := c.DialTimeout
|
|
|
|
if dialtime < 1 {
|
|
|
|
dialtime = 30
|
|
|
|
}
|
|
|
|
|
|
|
|
keepalivetime := c.KeepaliveTimeout
|
|
|
|
if keepalivetime < 1 {
|
|
|
|
keepalivetime = 1800
|
|
|
|
}
|
|
|
|
|
|
|
|
tlstime := c.TLSTimeout
|
|
|
|
if tlstime < 1 {
|
|
|
|
tlstime = 30
|
|
|
|
}
|
|
|
|
|
2017-04-28 15:21:38 +00:00
|
|
|
tr := &http.Transport{
|
|
|
|
Proxy: proxyFromClient(c),
|
|
|
|
TLSHandshakeTimeout: time.Duration(tlstime) * time.Second,
|
|
|
|
MaxIdleConnsPerHost: concurrentTransfers,
|
|
|
|
}
|
|
|
|
|
2017-04-27 23:16:01 +00:00
|
|
|
activityTimeout := 10
|
|
|
|
if v, ok := c.uc.Get("lfs", fmt.Sprintf("https://%v", host), "activitytimeout"); ok {
|
|
|
|
if i, err := strconv.Atoi(v); err == nil {
|
2017-04-28 15:21:38 +00:00
|
|
|
activityTimeout = i
|
|
|
|
} else {
|
|
|
|
activityTimeout = 0
|
2017-04-27 23:16:01 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
dialer := &net.Dialer{
|
|
|
|
Timeout: time.Duration(dialtime) * time.Second,
|
|
|
|
KeepAlive: time.Duration(keepalivetime) * time.Second,
|
|
|
|
DualStack: true,
|
|
|
|
}
|
|
|
|
|
2017-04-28 15:21:38 +00:00
|
|
|
if activityTimeout > 0 {
|
|
|
|
activityDuration := time.Duration(activityTimeout) * time.Second
|
|
|
|
tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
2017-04-27 23:16:01 +00:00
|
|
|
c, err := dialer.DialContext(ctx, network, addr)
|
|
|
|
if c == nil {
|
|
|
|
return c, err
|
|
|
|
}
|
|
|
|
if tc, ok := c.(*net.TCPConn); ok {
|
|
|
|
tc.SetKeepAlive(true)
|
|
|
|
tc.SetKeepAlivePeriod(dialer.KeepAlive)
|
|
|
|
}
|
|
|
|
return &deadlineConn{Timeout: activityDuration, Conn: c}, err
|
2017-04-28 15:21:38 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
tr.DialContext = dialer.DialContext
|
2016-12-20 22:37:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
tr.TLSClientConfig = &tls.Config{}
|
2017-01-16 14:58:06 +00:00
|
|
|
|
|
|
|
if isClientCertEnabledForHost(c, host) {
|
2017-01-20 09:50:33 +00:00
|
|
|
tracerx.Printf("http: client cert for %s", host)
|
2017-01-16 14:58:06 +00:00
|
|
|
tr.TLSClientConfig.Certificates = []tls.Certificate{getClientCertForHost(c, host)}
|
|
|
|
tr.TLSClientConfig.BuildNameToCertificate()
|
|
|
|
}
|
|
|
|
|
2016-12-20 22:37:11 +00:00
|
|
|
if isCertVerificationDisabledForHost(c, host) {
|
|
|
|
tr.TLSClientConfig.InsecureSkipVerify = true
|
|
|
|
} else {
|
|
|
|
tr.TLSClientConfig.RootCAs = getRootCAsForHost(c, host)
|
|
|
|
}
|
|
|
|
|
|
|
|
httpClient := &http.Client{
|
2016-12-21 17:06:55 +00:00
|
|
|
Transport: tr,
|
|
|
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
|
|
|
return http.ErrUseLastResponse
|
|
|
|
},
|
2016-12-20 22:37:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
c.hostClients[host] = httpClient
|
2016-12-22 18:01:51 +00:00
|
|
|
if c.VerboseOut == nil {
|
|
|
|
c.VerboseOut = os.Stderr
|
|
|
|
}
|
2016-12-20 22:37:11 +00:00
|
|
|
|
|
|
|
return httpClient
|
|
|
|
}
|
|
|
|
|
2017-01-03 20:38:15 +00:00
|
|
|
func (c *Client) CurrentUser() (string, string) {
|
|
|
|
userName, _ := c.gitEnv.Get("user.name")
|
|
|
|
userEmail, _ := c.gitEnv.Get("user.email")
|
|
|
|
return userName, userEmail
|
|
|
|
}
|
|
|
|
|
2016-12-21 17:06:55 +00:00
|
|
|
func newRequestForRetry(req *http.Request, location string) (*http.Request, error) {
|
|
|
|
newReq, err := http.NewRequest(req.Method, location, nil)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2016-12-20 22:37:11 +00:00
|
|
|
}
|
|
|
|
|
2017-05-19 16:18:03 +00:00
|
|
|
if req.URL.Scheme == "https" && newReq.URL.Scheme == "http" {
|
|
|
|
return nil, errors.New("lfsapi/client: refusing insecure redirect, https->http")
|
|
|
|
}
|
2017-05-19 14:18:58 +00:00
|
|
|
|
2017-05-19 16:18:03 +00:00
|
|
|
sameHost := req.URL.Host == newReq.URL.Host
|
2016-12-21 17:06:55 +00:00
|
|
|
for key := range req.Header {
|
2016-12-20 22:37:11 +00:00
|
|
|
if key == "Authorization" {
|
2017-05-19 16:18:03 +00:00
|
|
|
if !sameHost {
|
2016-12-20 22:37:11 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
2016-12-21 17:06:55 +00:00
|
|
|
newReq.Header.Set(key, req.Header.Get(key))
|
2016-12-20 22:37:11 +00:00
|
|
|
}
|
|
|
|
|
2016-12-21 17:06:55 +00:00
|
|
|
oldestURL := strings.SplitN(req.URL.String(), "?", 2)[0]
|
|
|
|
newURL := strings.SplitN(newReq.URL.String(), "?", 2)[0]
|
|
|
|
tracerx.Printf("api: redirect %s %s to %s", req.Method, oldestURL, newURL)
|
2016-12-20 22:37:11 +00:00
|
|
|
|
2017-05-18 21:04:59 +00:00
|
|
|
// This body will have already been rewound from a call to
|
|
|
|
// lfsapi.Client.traceRequest().
|
2016-12-21 17:06:55 +00:00
|
|
|
newReq.Body = req.Body
|
|
|
|
newReq.ContentLength = req.ContentLength
|
2017-08-24 21:22:22 +00:00
|
|
|
|
|
|
|
// Copy the request's context.Context, if any.
|
|
|
|
newReq = newReq.WithContext(req.Context())
|
|
|
|
|
2016-12-21 17:06:55 +00:00
|
|
|
return newReq, nil
|
2016-12-20 22:37:11 +00:00
|
|
|
}
|
2017-04-14 20:11:44 +00:00
|
|
|
|
2017-04-27 23:16:01 +00:00
|
|
|
type deadlineConn struct {
|
|
|
|
Timeout time.Duration
|
|
|
|
net.Conn
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *deadlineConn) Read(b []byte) (int, error) {
|
|
|
|
if err := c.Conn.SetDeadline(time.Now().Add(c.Timeout)); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
return c.Conn.Read(b)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *deadlineConn) Write(b []byte) (int, error) {
|
|
|
|
if err := c.Conn.SetDeadline(time.Now().Add(c.Timeout)); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return c.Conn.Write(b)
|
|
|
|
}
|
|
|
|
|
2017-04-14 20:11:44 +00:00
|
|
|
func init() {
|
|
|
|
UserAgent = config.VersionDesc
|
|
|
|
}
|