2016-05-13 16:38:06 +00:00
|
|
|
package httputil
|
2015-02-13 23:51:01 +00:00
|
|
|
|
|
|
|
import (
|
2015-06-03 21:33:00 +00:00
|
|
|
"bufio"
|
|
|
|
"bytes"
|
2015-02-13 23:51:01 +00:00
|
|
|
"crypto/tls"
|
2015-03-26 17:10:17 +00:00
|
|
|
"errors"
|
2015-03-05 19:49:15 +00:00
|
|
|
"fmt"
|
|
|
|
"io"
|
2015-04-10 18:43:58 +00:00
|
|
|
"net"
|
2015-02-13 23:51:01 +00:00
|
|
|
"net/http"
|
2015-06-03 21:33:00 +00:00
|
|
|
"net/http/httputil"
|
2015-02-13 23:51:01 +00:00
|
|
|
"os"
|
2015-06-05 15:13:57 +00:00
|
|
|
"path/filepath"
|
2015-03-05 19:49:15 +00:00
|
|
|
"strings"
|
2015-06-04 18:36:08 +00:00
|
|
|
"sync"
|
2015-04-10 18:43:58 +00:00
|
|
|
"time"
|
2015-05-13 19:43:41 +00:00
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
"github.com/github/git-lfs/config"
|
2015-05-25 18:20:50 +00:00
|
|
|
"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
|
2015-02-13 23:51:01 +00:00
|
|
|
)
|
|
|
|
|
2016-05-06 13:46:13 +00:00
|
|
|
type httpTransferStats struct {
|
2015-06-03 21:33:00 +00:00
|
|
|
HeaderSize int
|
|
|
|
BodySize int
|
|
|
|
Start time.Time
|
|
|
|
Stop time.Time
|
|
|
|
}
|
|
|
|
|
2016-05-06 13:46:13 +00:00
|
|
|
type httpTransfer struct {
|
|
|
|
requestStats *httpTransferStats
|
|
|
|
responseStats *httpTransferStats
|
2015-06-03 21:33:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
|
|
|
// TODO should use some locks
|
2016-05-06 13:46:13 +00:00
|
|
|
httpTransfers = make(map[*http.Response]*httpTransfer)
|
|
|
|
httpTransferBuckets = make(map[string][]*http.Response)
|
|
|
|
httpTransfersLock sync.Mutex
|
|
|
|
httpTransferBucketsLock sync.Mutex
|
2016-05-13 16:38:06 +00:00
|
|
|
httpClients map[string]*HttpClient
|
|
|
|
httpClientsMutex sync.Mutex
|
|
|
|
UserAgent string
|
2015-06-03 21:33:00 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func LogTransfer(key string, res *http.Response) {
|
2016-05-13 16:38:06 +00:00
|
|
|
if config.Config.IsLoggingStats {
|
2016-05-06 13:46:13 +00:00
|
|
|
httpTransferBucketsLock.Lock()
|
|
|
|
httpTransferBuckets[key] = append(httpTransferBuckets[key], res)
|
|
|
|
httpTransferBucketsLock.Unlock()
|
2015-06-04 18:30:47 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type HttpClient struct {
|
|
|
|
*http.Client
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *HttpClient) Do(req *http.Request) (*http.Response, error) {
|
|
|
|
traceHttpRequest(req)
|
|
|
|
|
|
|
|
crc := countingRequest(req)
|
|
|
|
if req.Body != nil {
|
|
|
|
// Only set the body if we have a body, but create the countingRequest
|
|
|
|
// anyway to make using zeroed stats easier.
|
|
|
|
req.Body = crc
|
|
|
|
}
|
|
|
|
|
|
|
|
start := time.Now()
|
|
|
|
res, err := c.Client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return res, err
|
|
|
|
}
|
|
|
|
|
|
|
|
traceHttpResponse(res)
|
|
|
|
|
|
|
|
cresp := countingResponse(res)
|
|
|
|
res.Body = cresp
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
if config.Config.IsLoggingStats {
|
2015-06-04 18:30:47 +00:00
|
|
|
reqHeaderSize := 0
|
2015-06-05 15:13:57 +00:00
|
|
|
resHeaderSize := 0
|
|
|
|
|
|
|
|
if dump, err := httputil.DumpRequest(req, false); err == nil {
|
2015-06-04 18:30:47 +00:00
|
|
|
reqHeaderSize = len(dump)
|
|
|
|
}
|
|
|
|
|
2015-06-05 15:13:57 +00:00
|
|
|
if dump, err := httputil.DumpResponse(res, false); err == nil {
|
|
|
|
resHeaderSize = len(dump)
|
|
|
|
}
|
|
|
|
|
2016-05-06 13:46:13 +00:00
|
|
|
reqstats := &httpTransferStats{HeaderSize: reqHeaderSize, BodySize: crc.Count}
|
2015-06-05 15:13:57 +00:00
|
|
|
|
|
|
|
// Response body size cannot be figured until it is read. Do not rely on a Content-Length
|
|
|
|
// header because it may not exist or be -1 in the case of chunked responses.
|
2016-05-06 13:46:13 +00:00
|
|
|
resstats := &httpTransferStats{HeaderSize: resHeaderSize, Start: start}
|
|
|
|
t := &httpTransfer{requestStats: reqstats, responseStats: resstats}
|
|
|
|
httpTransfersLock.Lock()
|
|
|
|
httpTransfers[res] = t
|
|
|
|
httpTransfersLock.Unlock()
|
2015-06-04 18:30:47 +00:00
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
|
|
|
|
return res, err
|
|
|
|
}
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
// NewHttpClient returns a new HttpClient for the given host (which may be "host:port")
|
|
|
|
func NewHttpClient(c *config.Configuration, host string) *HttpClient {
|
|
|
|
httpClientsMutex.Lock()
|
|
|
|
defer httpClientsMutex.Unlock()
|
|
|
|
|
|
|
|
if httpClients == nil {
|
|
|
|
httpClients = make(map[string]*HttpClient)
|
|
|
|
}
|
|
|
|
if client, ok := httpClients[host]; ok {
|
2016-03-11 09:19:23 +00:00
|
|
|
return client
|
2015-05-12 08:47:34 +00:00
|
|
|
}
|
|
|
|
|
2016-02-04 16:50:25 +00:00
|
|
|
dialtime := c.GitConfigInt("lfs.dialtimeout", 30)
|
|
|
|
keepalivetime := c.GitConfigInt("lfs.keepalive", 1800) // 30 minutes
|
|
|
|
tlstime := c.GitConfigInt("lfs.tlstimeout", 30)
|
|
|
|
|
2015-05-12 08:47:34 +00:00
|
|
|
tr := &http.Transport{
|
|
|
|
Proxy: http.ProxyFromEnvironment,
|
|
|
|
Dial: (&net.Dialer{
|
2016-02-04 16:50:25 +00:00
|
|
|
Timeout: time.Duration(dialtime) * time.Second,
|
|
|
|
KeepAlive: time.Duration(keepalivetime) * time.Second,
|
2015-05-12 08:47:34 +00:00
|
|
|
}).Dial,
|
2016-02-04 16:50:25 +00:00
|
|
|
TLSHandshakeTimeout: time.Duration(tlstime) * time.Second,
|
2016-02-04 16:52:57 +00:00
|
|
|
MaxIdleConnsPerHost: c.ConcurrentTransfers(),
|
2015-02-13 23:51:01 +00:00
|
|
|
}
|
2015-05-12 08:47:34 +00:00
|
|
|
|
2016-03-02 15:18:04 +00:00
|
|
|
tr.TLSClientConfig = &tls.Config{}
|
2016-03-17 16:33:48 +00:00
|
|
|
if isCertVerificationDisabledForHost(host) {
|
2016-03-02 15:18:04 +00:00
|
|
|
tr.TLSClientConfig.InsecureSkipVerify = true
|
|
|
|
} else {
|
2016-03-08 16:44:23 +00:00
|
|
|
tr.TLSClientConfig.RootCAs = getRootCAsForHost(host)
|
2015-05-12 08:47:34 +00:00
|
|
|
}
|
|
|
|
|
2016-03-11 09:19:23 +00:00
|
|
|
client := &HttpClient{
|
2016-05-13 16:38:06 +00:00
|
|
|
&http.Client{Transport: tr, CheckRedirect: CheckRedirect},
|
2015-05-12 08:47:34 +00:00
|
|
|
}
|
2016-05-13 16:38:06 +00:00
|
|
|
httpClients[host] = client
|
2015-05-12 08:47:34 +00:00
|
|
|
|
2016-03-11 09:19:23 +00:00
|
|
|
return client
|
2015-02-13 23:51:01 +00:00
|
|
|
}
|
2015-03-05 19:49:15 +00:00
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
func CheckRedirect(req *http.Request, via []*http.Request) error {
|
2015-03-26 17:10:17 +00:00
|
|
|
if len(via) >= 3 {
|
|
|
|
return errors.New("stopped after 3 redirects")
|
|
|
|
}
|
|
|
|
|
|
|
|
oldest := via[0]
|
|
|
|
for key, _ := range oldest.Header {
|
2015-03-26 17:19:59 +00:00
|
|
|
if key == "Authorization" {
|
|
|
|
if req.URL.Scheme != oldest.URL.Scheme || req.URL.Host != oldest.URL.Host {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
2015-03-26 17:10:17 +00:00
|
|
|
req.Header.Set(key, oldest.Header.Get(key))
|
|
|
|
}
|
|
|
|
|
2016-01-29 21:52:55 +00:00
|
|
|
oldestUrl := strings.SplitN(oldest.URL.String(), "?", 2)[0]
|
|
|
|
newUrl := strings.SplitN(req.URL.String(), "?", 2)[0]
|
|
|
|
tracerx.Printf("api: redirect %s %s to %s", oldest.Method, oldestUrl, newUrl)
|
2015-03-26 18:46:33 +00:00
|
|
|
|
2015-03-26 17:10:17 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2015-03-05 19:49:15 +00:00
|
|
|
var tracedTypes = []string{"json", "text", "xml", "html"}
|
|
|
|
|
2015-06-03 21:33:00 +00:00
|
|
|
func traceHttpRequest(req *http.Request) {
|
2016-05-13 16:38:06 +00:00
|
|
|
tracerx.Printf("HTTP: %s", TraceHttpReq(req))
|
2015-03-05 19:49:15 +00:00
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
if config.Config.IsTracingHttp == false {
|
2015-03-05 19:49:15 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2015-06-03 21:33:00 +00:00
|
|
|
dump, err := httputil.DumpRequest(req, false)
|
|
|
|
if err != nil {
|
|
|
|
return
|
2015-03-22 20:13:37 +00:00
|
|
|
}
|
|
|
|
|
2015-11-18 16:41:29 +00:00
|
|
|
traceHttpDump(">", dump)
|
2015-03-05 19:49:15 +00:00
|
|
|
}
|
|
|
|
|
2015-06-03 21:33:00 +00:00
|
|
|
func traceHttpResponse(res *http.Response) {
|
2015-03-26 18:46:33 +00:00
|
|
|
if res == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2015-03-05 19:49:15 +00:00
|
|
|
tracerx.Printf("HTTP: %d", res.StatusCode)
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
if config.Config.IsTracingHttp == false {
|
2015-03-05 19:49:15 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2015-06-03 21:33:00 +00:00
|
|
|
dump, err := httputil.DumpResponse(res, false)
|
|
|
|
if err != nil {
|
|
|
|
return
|
2015-03-05 19:49:15 +00:00
|
|
|
}
|
|
|
|
|
2015-11-18 01:05:37 +00:00
|
|
|
if isTraceableContent(res.Header) {
|
|
|
|
fmt.Fprintf(os.Stderr, "\n\n")
|
|
|
|
} else {
|
|
|
|
fmt.Fprintf(os.Stderr, "\n")
|
|
|
|
}
|
|
|
|
|
2015-11-18 16:41:29 +00:00
|
|
|
traceHttpDump("<", dump)
|
2015-11-18 01:05:37 +00:00
|
|
|
}
|
|
|
|
|
2015-11-18 16:41:29 +00:00
|
|
|
func traceHttpDump(direction string, dump []byte) {
|
2015-06-03 21:33:00 +00:00
|
|
|
scanner := bufio.NewScanner(bytes.NewBuffer(dump))
|
2015-11-18 01:05:37 +00:00
|
|
|
|
2015-06-03 21:33:00 +00:00
|
|
|
for scanner.Scan() {
|
2015-11-18 01:05:37 +00:00
|
|
|
line := scanner.Text()
|
2016-05-13 16:38:06 +00:00
|
|
|
if !config.Config.IsDebuggingHttp && strings.HasPrefix(strings.ToLower(line), "authorization: basic") {
|
2015-11-18 16:41:29 +00:00
|
|
|
fmt.Fprintf(os.Stderr, "%s Authorization: Basic * * * * *\n", direction)
|
2015-11-18 01:05:37 +00:00
|
|
|
} else {
|
2015-11-18 16:41:29 +00:00
|
|
|
fmt.Fprintf(os.Stderr, "%s %s\n", direction, line)
|
2015-11-18 01:05:37 +00:00
|
|
|
}
|
2015-03-05 19:49:15 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-11-18 01:05:37 +00:00
|
|
|
func isTraceableContent(h http.Header) bool {
|
|
|
|
ctype := strings.ToLower(strings.SplitN(h.Get("Content-Type"), ";", 2)[0])
|
|
|
|
for _, tracedType := range tracedTypes {
|
|
|
|
if strings.Contains(ctype, tracedType) {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
func countingRequest(req *http.Request) *CountingReadCloser {
|
|
|
|
return &CountingReadCloser{
|
2015-11-18 17:51:06 +00:00
|
|
|
request: req,
|
|
|
|
ReadCloser: req.Body,
|
|
|
|
isTraceableType: isTraceableContent(req.Header),
|
|
|
|
useGitTrace: false,
|
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
}
|
2015-03-22 18:30:54 +00:00
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
func countingResponse(res *http.Response) *CountingReadCloser {
|
|
|
|
return &CountingReadCloser{
|
2015-11-18 17:51:06 +00:00
|
|
|
response: res,
|
|
|
|
ReadCloser: res.Body,
|
|
|
|
isTraceableType: isTraceableContent(res.Header),
|
|
|
|
useGitTrace: true,
|
|
|
|
}
|
2015-03-05 21:18:02 +00:00
|
|
|
}
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
type CountingReadCloser struct {
|
2015-11-18 01:05:37 +00:00
|
|
|
Count int
|
|
|
|
request *http.Request
|
|
|
|
response *http.Response
|
|
|
|
isTraceableType bool
|
2015-11-18 17:51:06 +00:00
|
|
|
useGitTrace bool
|
2015-06-03 21:33:00 +00:00
|
|
|
io.ReadCloser
|
2015-03-05 21:18:02 +00:00
|
|
|
}
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
func (c *CountingReadCloser) Read(b []byte) (int, error) {
|
2015-06-03 21:33:00 +00:00
|
|
|
n, err := c.ReadCloser.Read(b)
|
|
|
|
if err != nil && err != io.EOF {
|
|
|
|
return n, err
|
2015-03-22 18:30:54 +00:00
|
|
|
}
|
2015-03-05 21:18:02 +00:00
|
|
|
|
2015-06-03 21:33:00 +00:00
|
|
|
c.Count += n
|
2015-03-22 18:30:54 +00:00
|
|
|
|
2015-11-18 17:51:06 +00:00
|
|
|
if c.isTraceableType {
|
|
|
|
chunk := string(b[0:n])
|
|
|
|
if c.useGitTrace {
|
|
|
|
tracerx.Printf("HTTP: %s", chunk)
|
|
|
|
}
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
if config.Config.IsTracingHttp {
|
2015-11-18 17:51:06 +00:00
|
|
|
fmt.Fprint(os.Stderr, chunk)
|
|
|
|
}
|
2015-06-03 21:33:00 +00:00
|
|
|
}
|
2015-03-05 19:49:15 +00:00
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
if err == io.EOF && config.Config.IsLoggingStats {
|
2016-05-06 13:46:13 +00:00
|
|
|
// This httpTransfer is done, we're checking it this way so we can also
|
|
|
|
// catch httpTransfers where the caller forgets to Close() the Body.
|
2015-06-03 21:33:00 +00:00
|
|
|
if c.response != nil {
|
2016-05-06 13:46:13 +00:00
|
|
|
httpTransfersLock.Lock()
|
|
|
|
if httpTransfer, ok := httpTransfers[c.response]; ok {
|
|
|
|
httpTransfer.responseStats.BodySize = c.Count
|
|
|
|
httpTransfer.responseStats.Stop = time.Now()
|
2015-06-03 21:33:00 +00:00
|
|
|
}
|
2016-05-06 13:46:13 +00:00
|
|
|
httpTransfersLock.Unlock()
|
2015-06-03 21:33:00 +00:00
|
|
|
}
|
|
|
|
}
|
2015-03-05 19:49:15 +00:00
|
|
|
return n, err
|
|
|
|
}
|
|
|
|
|
2015-06-05 15:13:57 +00:00
|
|
|
// LogHttpStats is intended to be called after all HTTP operations for the
|
2016-05-06 13:46:13 +00:00
|
|
|
// commmand have finished. It dumps k/v logs, one line per httpTransfer into
|
2015-06-05 15:13:57 +00:00
|
|
|
// a log file with the current timestamp.
|
2015-06-04 19:18:14 +00:00
|
|
|
func LogHttpStats() {
|
2016-05-13 16:38:06 +00:00
|
|
|
if !config.Config.IsLoggingStats {
|
2015-06-04 19:18:14 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2015-06-05 15:13:57 +00:00
|
|
|
file, err := statsLogFile()
|
2015-06-04 19:18:14 +00:00
|
|
|
if err != nil {
|
|
|
|
fmt.Fprintf(os.Stderr, "Error logging http stats: %s\n", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
fmt.Fprintf(file, "concurrent=%d batch=%v time=%d version=%s\n", config.Config.ConcurrentTransfers(), config.Config.BatchTransfer(), time.Now().Unix(), config.Version)
|
2015-06-05 16:22:12 +00:00
|
|
|
|
2016-05-06 13:46:13 +00:00
|
|
|
for key, responses := range httpTransferBuckets {
|
2015-06-04 19:18:14 +00:00
|
|
|
for _, response := range responses {
|
2016-05-06 13:46:13 +00:00
|
|
|
stats := httpTransfers[response]
|
2015-06-05 16:32:20 +00:00
|
|
|
fmt.Fprintf(file, "key=%s reqheader=%d reqbody=%d resheader=%d resbody=%d restime=%d status=%d url=%s\n",
|
2015-06-04 19:18:14 +00:00
|
|
|
key,
|
|
|
|
stats.requestStats.HeaderSize,
|
|
|
|
stats.requestStats.BodySize,
|
|
|
|
stats.responseStats.HeaderSize,
|
|
|
|
stats.responseStats.BodySize,
|
|
|
|
stats.responseStats.Stop.Sub(stats.responseStats.Start).Nanoseconds(),
|
2015-06-05 16:32:20 +00:00
|
|
|
response.StatusCode,
|
2015-06-16 22:34:41 +00:00
|
|
|
response.Request.URL)
|
2015-06-04 19:18:14 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fmt.Fprintf(os.Stderr, "HTTP Stats logged to file %s\n", file.Name())
|
|
|
|
}
|
|
|
|
|
2015-06-05 15:13:57 +00:00
|
|
|
func statsLogFile() (*os.File, error) {
|
2016-05-13 16:38:06 +00:00
|
|
|
logBase := filepath.Join(config.LocalLogDir, "http")
|
2015-06-05 16:37:00 +00:00
|
|
|
if err := os.MkdirAll(logBase, 0755); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2015-06-05 15:13:57 +00:00
|
|
|
logFile := fmt.Sprintf("http-%d.log", time.Now().Unix())
|
2015-06-05 16:37:00 +00:00
|
|
|
return os.Create(filepath.Join(logBase, logFile))
|
2015-03-05 19:49:15 +00:00
|
|
|
}
|
2016-05-13 16:38:06 +00:00
|
|
|
|
|
|
|
func TraceHttpReq(req *http.Request) string {
|
|
|
|
return fmt.Sprintf("%s %s", req.Method, strings.SplitN(req.URL.String(), "?", 2)[0])
|
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
UserAgent = config.VersionDesc
|
|
|
|
}
|