lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replace config.Config references in httputil

Browse Source
This commit is contained in:
risk danger olson 2016-07-21 18:07:23 -06:00
parent 4f71d45acd
commit b9c5a108d9
18 changed files with 181 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
api/api.go
View File

@ -21,7 +21,8 @@ import (
// This is for simplicity, legacy route is not most optimal (serial)
// TODO LEGACY API: remove when legacy API removed
func BatchOrLegacy(objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
if !config.Config.BatchTransfer() {
cfg := config.Config
if !cfg.BatchTransfer() {
objs, err := Legacy(objects, operation)
return objs, "", err
}
@ -54,6 +55,8 @@ func Batch(objects []*ObjectResource, operation string, transferAdapters []strin
return nil, "", nil
}
cfg := config.Config
o := &batchRequest{Operation: operation, Objects: objects, TransferAdapterNames: transferAdapters}
by, err := json.Marshal(o)
if err != nil {
@ -85,7 +88,7 @@ func Batch(objects []*ObjectResource, operation string, transferAdapters []strin
}
if errutil.IsAuthError(err) {
httputil.SetAuthType(req, res)
httputil.SetAuthType(cfg, req, res)
return Batch(objects, operation, transferAdapters)
}
@ -98,7 +101,7 @@ func Batch(objects []*ObjectResource, operation string, transferAdapters []strin
tracerx.Printf("api error: %s", err)
return nil, "", errutil.Error(err)
}
httputil.LogTransfer("lfs.batch", res)
httputil.LogTransfer(cfg, "lfs.batch", res)
if res.StatusCode != 200 {
return nil, "", errutil.Error(fmt.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode))
@ -141,7 +144,9 @@ func DownloadCheck(oid string) (*ObjectResource, error) {
if err != nil {
return nil, err
}
httputil.LogTransfer("lfs.download", res)
cfg := config.Config
httputil.LogTransfer(cfg, "lfs.download", res)
_, err = obj.NewRequest("download", "GET")
if err != nil {
@ -153,7 +158,7 @@ func DownloadCheck(oid string) (*ObjectResource, error) {
// TODO LEGACY API: remove when legacy API removed
func UploadCheck(oid string, size int64) (*ObjectResource, error) {
cfg := config.Config
reqObj := &ObjectResource{
Oid: oid,
Size: size,
@ -179,13 +184,13 @@ func UploadCheck(oid string, size int64) (*ObjectResource, error) {
if err != nil {
if errutil.IsAuthError(err) {
httputil.SetAuthType(req, res)
httputil.SetAuthType(cfg, req, res)
return UploadCheck(oid, size)
}
return nil, errutil.NewRetriableError(err)
}
httputil.LogTransfer("lfs.upload", res)
httputil.LogTransfer(cfg, "lfs.upload", res)
if res.StatusCode == 200 {
return nil, nil

2
api/http_lifecycle.go
View File

@ -102,7 +102,7 @@ func (l *HttpLifecycle) Build(schema *RequestSchema) (*http.Request, error) {
// Otherwise, the api.Response is returned, along with no error, signaling that
// the request completed successfully.
func (l *HttpLifecycle) Execute(req *http.Request, into interface{}) (Response, error) {
resp, err := httputil.DoHttpRequestWithRedirects(req, []*http.Request{}, true)
resp, err := httputil.DoHttpRequestWithRedirects(config.Config, req, []*http.Request{}, true)
if err != nil {
return nil, err
}

12
api/v1.go
View File

@ -19,8 +19,9 @@ const (
// doLegacyApiRequest runs the request to the LFS legacy API.
func DoLegacyRequest(req *http.Request) (*http.Response, *ObjectResource, error) {
cfg := config.Config
via := make([]*http.Request, 0, 4)
res, err := httputil.DoHttpRequestWithRedirects(req, via, true)
res, err := httputil.DoHttpRequestWithRedirects(cfg, req, via, true)
if err != nil {
return res, nil, err
}
@ -29,7 +30,7 @@ func DoLegacyRequest(req *http.Request) (*http.Response, *ObjectResource, error)
err = httputil.DecodeResponse(res, obj)
if err != nil {
httputil.SetErrorResponseContext(err, res)
httputil.SetErrorResponseContext(cfg, err, res)
return nil, nil, err
}
@ -51,7 +52,8 @@ type batchResponse struct {
// re-run. When the repo is marked as having private access, credentials will
// be retrieved.
func DoBatchRequest(req *http.Request) (*http.Response, *batchResponse, error) {
res, err := DoRequest(req, config.Config.PrivateAccess(auth.GetOperationForRequest(req)))
cfg := config.Config
res, err := DoRequest(req, cfg.PrivateAccess(auth.GetOperationForRequest(req)))
if err != nil {
if res != nil && res.StatusCode == 401 {
@ -64,7 +66,7 @@ func DoBatchRequest(req *http.Request) (*http.Response, *batchResponse, error) {
err = httputil.DecodeResponse(res, resp)
if err != nil {
httputil.SetErrorResponseContext(err, res)
httputil.SetErrorResponseContext(cfg, err, res)
}
return res, resp, err
@ -76,7 +78,7 @@ func DoBatchRequest(req *http.Request) (*http.Response, *batchResponse, error) {
// private access, credentials will be retrieved.
func DoRequest(req *http.Request, useCreds bool) (*http.Response, error) {
via := make([]*http.Request, 0, 4)
return httputil.DoHttpRequestWithRedirects(req, via, useCreds)
return httputil.DoHttpRequestWithRedirects(config.Config, req, via, useCreds)
}
func NewRequest(method, oid string) (*http.Request, error) {

5
api/verify.go
View File

@ -7,13 +7,13 @@ import (
"io/ioutil"
"strconv"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/httputil"
)
// VerifyUpload calls the "verify" API link relation on obj if it exists
func VerifyUpload(obj *ObjectResource) error {
// Do we need to do verify?
if _, ok := obj.Rel("verify"); !ok {
return nil
@ -38,7 +38,8 @@ func VerifyUpload(obj *ObjectResource) error {
return err
}
httputil.LogTransfer("lfs.data.verify", res)
cfg := config.Config
httputil.LogTransfer(cfg, "lfs.data.verify", res)
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()

8
commands/commands.go
View File

@ -15,6 +15,7 @@ import (
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/git"
"github.com/github/git-lfs/httputil"
"github.com/github/git-lfs/lfs"
"github.com/github/git-lfs/tools"
"github.com/spf13/cobra"
@ -106,6 +107,13 @@ func Panic(err error, format string, args ...interface{}) {
func Run() {
RootCmd.Execute()
httputil.LogHttpStats(cfg)
}
func Cleanup() {
if err := lfs.ClearTempObjects(); err != nil {
fmt.Fprintf(os.Stderr, "Error clearing old temp files: %s\n", err)
}
}
func PipeMediaCommand(name string, args ...string) error {

13
git-lfs.go
View File

@ -8,8 +8,6 @@ import (
"syscall"
"github.com/github/git-lfs/commands"
"github.com/github/git-lfs/httputil"
"github.com/github/git-lfs/lfs"
)
func main() {
@ -21,7 +19,7 @@ func main() {
go func() {
for {
sig := <-c
once.Do(clearTempObjects)
once.Do(commands.Cleanup)
fmt.Fprintf(os.Stderr, "\nExiting because of %q signal.\n", sig)
exitCode := 1
@ -33,12 +31,5 @@ func main() {
}()
commands.Run()
httputil.LogHttpStats()
once.Do(clearTempObjects)
}
func clearTempObjects() {
if err := lfs.ClearTempObjects(); err != nil {
fmt.Fprintf(os.Stderr, "Error clearing old temp files: %s\n", err)
}
once.Do(commands.Cleanup)
}

24
httputil/certs.go
View File

@ -12,14 +12,14 @@ import (
// isCertVerificationDisabledForHost returns whether SSL certificate verification
// has been disabled for the given host, or globally
func isCertVerificationDisabledForHost(host string) bool {
hostSslVerify, _ := config.Config.GitConfig(fmt.Sprintf("http.https://%v/.sslverify", host))
func isCertVerificationDisabledForHost(cfg *config.Configuration, host string) bool {
hostSslVerify, _ := cfg.GitConfig(fmt.Sprintf("http.https://%v/.sslverify", host))
if hostSslVerify == "false" {
return true
}
globalSslVerify, _ := config.Config.GitConfig("http.sslverify")
if globalSslVerify == "false" || config.Config.GetenvBool("GIT_SSL_NO_VERIFY", false) {
globalSslVerify, _ := cfg.GitConfig("http.sslverify")
if globalSslVerify == "false" || cfg.GetenvBool("GIT_SSL_NO_VERIFY", false) {
return true
}
@ -32,41 +32,41 @@ func isCertVerificationDisabledForHost(host string) bool {
// source which is not included by default in the golang certificate search)
// May return nil if it doesn't have anything to add, in which case the default
// RootCAs will be used if passed to TLSClientConfig.RootCAs
func getRootCAsForHost(host string) *x509.CertPool {
func getRootCAsForHost(cfg *config.Configuration, host string) *x509.CertPool {
// don't init pool, want to return nil not empty if none found; init only on successful add cert
var pool *x509.CertPool
// gitconfig first
pool = appendRootCAsForHostFromGitconfig(pool, host)
pool = appendRootCAsForHostFromGitconfig(cfg, pool, host)
// Platform specific
return appendRootCAsForHostFromPlatform(pool, host)
}
func appendRootCAsForHostFromGitconfig(pool *x509.CertPool, host string) *x509.CertPool {
func appendRootCAsForHostFromGitconfig(cfg *config.Configuration, pool *x509.CertPool, host string) *x509.CertPool {
// Accumulate certs from all these locations:
// GIT_SSL_CAINFO first
if cafile := config.Config.Getenv("GIT_SSL_CAINFO"); len(cafile) > 0 {
if cafile := cfg.Getenv("GIT_SSL_CAINFO"); len(cafile) > 0 {
return appendCertsFromFile(pool, cafile)
}
// http.<url>.sslcainfo
// we know we have simply "host" or "host:port"
key := fmt.Sprintf("http.https://%v/.sslcainfo", host)
if cafile, ok := config.Config.GitConfig(key); ok {
if cafile, ok := cfg.GitConfig(key); ok {
return appendCertsFromFile(pool, cafile)
}
// http.sslcainfo
if cafile, ok := config.Config.GitConfig("http.sslcainfo"); ok {
if cafile, ok := cfg.GitConfig("http.sslcainfo"); ok {
return appendCertsFromFile(pool, cafile)
}
// GIT_SSL_CAPATH
if cadir := config.Config.Getenv("GIT_SSL_CAPATH"); len(cadir) > 0 {
if cadir := cfg.Getenv("GIT_SSL_CAPATH"); len(cadir) > 0 {
return appendCertsFromFilesInDir(pool, cadir)
}
// http.sslcapath
if cadir, ok := config.Config.GitConfig("http.sslcapath"); ok {
if cadir, ok := cfg.GitConfig("http.sslcapath"); ok {
return appendCertsFromFilesInDir(pool, cadir)
}

109
httputil/certs_test.go
View File

@ -35,8 +35,6 @@ NpW5Hh0w4/5iIetCkJ0=
-----END CERTIFICATE-----`
func TestCertFromSSLCAInfoConfig(t *testing.T) {
defer config.Config.ResetConfig()
tempfile, err := ioutil.TempFile("", "testcert")
assert.Nil(t, err, "Error creating temp cert file")
defer os.Remove(tempfile.Name())
@ -45,37 +43,34 @@ func TestCertFromSSLCAInfoConfig(t *testing.T) {
assert.Nil(t, err, "Error writing temp cert file")
tempfile.Close()
config.Config.ClearConfig()
config.Config.SetConfig("http.https://git-lfs.local/.sslcainfo", tempfile.Name())
cfg := config.New()
cfg.SetConfig("http.https://git-lfs.local/.sslcainfo", tempfile.Name())
// Should match
pool := getRootCAsForHost("git-lfs.local")
t.Log("Should match")
pool := getRootCAsForHost(cfg, "git-lfs.local")
assert.NotNil(t, pool)
// Shouldn't match
pool = getRootCAsForHost("wronghost.com")
t.Log("Should not match")
pool = getRootCAsForHost(cfg, "wronghost.com")
assert.Nil(t, pool)
// Ports have to match
pool = getRootCAsForHost("git-lfs.local:8443")
t.Log("Ports have to match")
pool = getRootCAsForHost(cfg, "git-lfs.local:8443")
assert.Nil(t, pool)
// Now use global sslcainfo
config.Config.ClearConfig()
config.Config.SetConfig("http.sslcainfo", tempfile.Name())
cfg.SetConfig("http.sslcainfo", tempfile.Name())
// Should match anything
pool = getRootCAsForHost("git-lfs.local")
t.Log("Should match anything")
pool = getRootCAsForHost(cfg, "git-lfs.local")
assert.NotNil(t, pool)
pool = getRootCAsForHost("wronghost.com")
pool = getRootCAsForHost(cfg, "wronghost.com")
assert.NotNil(t, pool)
pool = getRootCAsForHost("git-lfs.local:8443")
pool = getRootCAsForHost(cfg, "git-lfs.local:8443")
assert.NotNil(t, pool)
}
func TestCertFromSSLCAInfoEnv(t *testing.T) {
tempfile, err := ioutil.TempFile("", "testcert")
assert.Nil(t, err, "Error creating temp cert file")
defer os.Remove(tempfile.Name())
@ -84,25 +79,20 @@ func TestCertFromSSLCAInfoEnv(t *testing.T) {
assert.Nil(t, err, "Error writing temp cert file")
tempfile.Close()
oldEnv := config.Config.GetAllEnv()
defer func() {
config.Config.SetAllEnv(oldEnv)
}()
config.Config.SetAllEnv(map[string]string{"GIT_SSL_CAINFO": tempfile.Name()})
cfg := config.New()
cfg.SetAllEnv(map[string]string{"GIT_SSL_CAINFO": tempfile.Name()})
// Should match any host at all
pool := getRootCAsForHost("git-lfs.local")
pool := getRootCAsForHost(cfg, "git-lfs.local")
assert.NotNil(t, pool)
pool = getRootCAsForHost("wronghost.com")
pool = getRootCAsForHost(cfg, "wronghost.com")
assert.NotNil(t, pool)
pool = getRootCAsForHost("notthisone.com:8888")
pool = getRootCAsForHost(cfg, "notthisone.com:8888")
assert.NotNil(t, pool)
}
func TestCertFromSSLCAPathConfig(t *testing.T) {
defer config.Config.ResetConfig()
tempdir, err := ioutil.TempDir("", "testcertdir")
assert.Nil(t, err, "Error creating temp cert dir")
defer os.RemoveAll(tempdir)
@ -110,21 +100,20 @@ func TestCertFromSSLCAPathConfig(t *testing.T) {
err = ioutil.WriteFile(filepath.Join(tempdir, "cert1.pem"), []byte(testCert), 0644)
assert.Nil(t, err, "Error creating cert file")
config.Config.ClearConfig()
config.Config.SetConfig("http.sslcapath", tempdir)
cfg := config.New()
cfg.SetConfig("http.sslcapath", tempdir)
// Should match any host at all
pool := getRootCAsForHost("git-lfs.local")
pool := getRootCAsForHost(cfg, "git-lfs.local")
assert.NotNil(t, pool)
pool = getRootCAsForHost("wronghost.com")
pool = getRootCAsForHost(cfg, "wronghost.com")
assert.NotNil(t, pool)
pool = getRootCAsForHost("notthisone.com:8888")
pool = getRootCAsForHost(cfg, "notthisone.com:8888")
assert.NotNil(t, pool)
}
func TestCertFromSSLCAPathEnv(t *testing.T) {
tempdir, err := ioutil.TempDir("", "testcertdir")
assert.Nil(t, err, "Error creating temp cert dir")
defer os.RemoveAll(tempdir)
@ -132,55 +121,41 @@ func TestCertFromSSLCAPathEnv(t *testing.T) {
err = ioutil.WriteFile(filepath.Join(tempdir, "cert1.pem"), []byte(testCert), 0644)
assert.Nil(t, err, "Error creating cert file")
oldEnv := config.Config.GetAllEnv()
defer func() {
config.Config.SetAllEnv(oldEnv)
}()
config.Config.SetAllEnv(map[string]string{"GIT_SSL_CAPATH": tempdir})
cfg := config.New()
cfg.SetAllEnv(map[string]string{"GIT_SSL_CAPATH": tempdir})
// Should match any host at all
pool := getRootCAsForHost("git-lfs.local")
pool := getRootCAsForHost(cfg, "git-lfs.local")
assert.NotNil(t, pool)
pool = getRootCAsForHost("wronghost.com")
pool = getRootCAsForHost(cfg, "wronghost.com")
assert.NotNil(t, pool)
pool = getRootCAsForHost("notthisone.com:8888")
pool = getRootCAsForHost(cfg, "notthisone.com:8888")
assert.NotNil(t, pool)
}
func TestCertVerifyDisabledGlobalEnv(t *testing.T) {
cfg := config.New()
assert.False(t, isCertVerificationDisabledForHost(cfg, "anyhost.com"))
assert.False(t, isCertVerificationDisabledForHost("anyhost.com"))
oldEnv := config.Config.GetAllEnv()
defer func() {
config.Config.SetAllEnv(oldEnv)
}()
config.Config.SetAllEnv(map[string]string{"GIT_SSL_NO_VERIFY": "1"})
assert.True(t, isCertVerificationDisabledForHost("anyhost.com"))
cfg.SetAllEnv(map[string]string{"GIT_SSL_NO_VERIFY": "1"})
assert.True(t, isCertVerificationDisabledForHost(cfg, "anyhost.com"))
}
func TestCertVerifyDisabledGlobalConfig(t *testing.T) {
defer config.Config.ResetConfig()
cfg := config.New()
assert.False(t, isCertVerificationDisabledForHost(cfg, "anyhost.com"))
assert.False(t, isCertVerificationDisabledForHost("anyhost.com"))
config.Config.ClearConfig()
config.Config.SetConfig("http.sslverify", "false")
assert.True(t, isCertVerificationDisabledForHost("anyhost.com"))
cfg.SetConfig("http.sslverify", "false")
assert.True(t, isCertVerificationDisabledForHost(cfg, "anyhost.com"))
}
func TestCertVerifyDisabledHostConfig(t *testing.T) {
defer config.Config.ResetConfig()
cfg := config.New()
assert.False(t, isCertVerificationDisabledForHost(cfg, "specifichost.com"))
assert.False(t, isCertVerificationDisabledForHost(cfg, "otherhost.com"))
assert.False(t, isCertVerificationDisabledForHost("specifichost.com"))
assert.False(t, isCertVerificationDisabledForHost("otherhost.com"))
config.Config.ClearConfig()
config.Config.SetConfig("http.https://specifichost.com/.sslverify", "false")
assert.True(t, isCertVerificationDisabledForHost("specifichost.com"))
assert.False(t, isCertVerificationDisabledForHost("otherhost.com"))
cfg.SetConfig("http.https://specifichost.com/.sslverify", "false")
assert.True(t, isCertVerificationDisabledForHost(cfg, "specifichost.com"))
assert.False(t, isCertVerificationDisabledForHost(cfg, "otherhost.com"))
}

55
httputil/http.go
View File

@ -45,8 +45,8 @@ var (
UserAgent string
)
func LogTransfer(key string, res *http.Response) {
if config.Config.IsLoggingStats {
func LogTransfer(cfg *config.Configuration, key string, res *http.Response) {
if cfg.IsLoggingStats {
httpTransferBucketsLock.Lock()
httpTransferBuckets[key] = append(httpTransferBuckets[key], res)
httpTransferBucketsLock.Unlock()
@ -54,13 +54,14 @@ func LogTransfer(key string, res *http.Response) {
}
type HttpClient struct {
Config *config.Configuration
*http.Client
}
func (c *HttpClient) Do(req *http.Request) (*http.Response, error) {
traceHttpRequest(req)
traceHttpRequest(c.Config, req)
crc := countingRequest(req)
crc := countingRequest(c.Config, req)
if req.Body != nil {
// Only set the body if we have a body, but create the countingRequest
// anyway to make using zeroed stats easier.
@ -73,12 +74,12 @@ func (c *HttpClient) Do(req *http.Request) (*http.Response, error) {
return res, err
}
traceHttpResponse(res)
traceHttpResponse(c.Config, res)
cresp := countingResponse(res)
cresp := countingResponse(c.Config, res)
res.Body = cresp
if config.Config.IsLoggingStats {
if c.Config.IsLoggingStats {
reqHeaderSize := 0
resHeaderSize := 0
@ -131,14 +132,15 @@ func NewHttpClient(c *config.Configuration, host string) *HttpClient {
}
tr.TLSClientConfig = &tls.Config{}
if isCertVerificationDisabledForHost(host) {
if isCertVerificationDisabledForHost(c, host) {
tr.TLSClientConfig.InsecureSkipVerify = true
} else {
tr.TLSClientConfig.RootCAs = getRootCAsForHost(host)
tr.TLSClientConfig.RootCAs = getRootCAsForHost(c, host)
}
client := &HttpClient{
&http.Client{Transport: tr, CheckRedirect: CheckRedirect},
Config: c,
Client: &http.Client{Transport: tr, CheckRedirect: CheckRedirect},
}
httpClients[host] = client
@ -169,10 +171,10 @@ func CheckRedirect(req *http.Request, via []*http.Request) error {
var tracedTypes = []string{"json", "text", "xml", "html"}
func traceHttpRequest(req *http.Request) {
func traceHttpRequest(cfg *config.Configuration, req *http.Request) {
tracerx.Printf("HTTP: %s", TraceHttpReq(req))
if config.Config.IsTracingHttp == false {
if cfg.IsTracingHttp == false {
return
}
@ -181,17 +183,17 @@ func traceHttpRequest(req *http.Request) {
return
}
traceHttpDump(">", dump)
traceHttpDump(cfg, ">", dump)
}
func traceHttpResponse(res *http.Response) {
func traceHttpResponse(cfg *config.Configuration, res *http.Response) {
if res == nil {
return
}
tracerx.Printf("HTTP: %d", res.StatusCode)
if config.Config.IsTracingHttp == false {
if cfg.IsTracingHttp == false {
return
}
@ -206,15 +208,15 @@ func traceHttpResponse(res *http.Response) {
fmt.Fprintf(os.Stderr, "\n")
}
traceHttpDump("<", dump)
traceHttpDump(cfg, "<", dump)
}
func traceHttpDump(direction string, dump []byte) {
func traceHttpDump(cfg *config.Configuration, direction string, dump []byte) {
scanner := bufio.NewScanner(bytes.NewBuffer(dump))
for scanner.Scan() {
line := scanner.Text()
if !config.Config.IsDebuggingHttp && strings.HasPrefix(strings.ToLower(line), "authorization: basic") {
if !cfg.IsDebuggingHttp && strings.HasPrefix(strings.ToLower(line), "authorization: basic") {
fmt.Fprintf(os.Stderr, "%s Authorization: Basic * * * * *\n", direction)
} else {
fmt.Fprintf(os.Stderr, "%s %s\n", direction, line)
@ -232,18 +234,20 @@ func isTraceableContent(h http.Header) bool {
return false
}
func countingRequest(req *http.Request) *CountingReadCloser {
func countingRequest(cfg *config.Configuration, req *http.Request) *CountingReadCloser {
return &CountingReadCloser{
request: req,
cfg: cfg,
ReadCloser: req.Body,
isTraceableType: isTraceableContent(req.Header),
useGitTrace: false,
}
}
func countingResponse(res *http.Response) *CountingReadCloser {
func countingResponse(cfg *config.Configuration, res *http.Response) *CountingReadCloser {
return &CountingReadCloser{
response: res,
cfg: cfg,
ReadCloser: res.Body,
isTraceableType: isTraceableContent(res.Header),
useGitTrace: true,
@ -254,6 +258,7 @@ type CountingReadCloser struct {
Count int
request *http.Request
response *http.Response
cfg *config.Configuration
isTraceableType bool
useGitTrace bool
io.ReadCloser
@ -273,12 +278,12 @@ func (c *CountingReadCloser) Read(b []byte) (int, error) {
tracerx.Printf("HTTP: %s", chunk)
}
if config.Config.IsTracingHttp {
if c.cfg.IsTracingHttp {
fmt.Fprint(os.Stderr, chunk)
}
}
if err == io.EOF && config.Config.IsLoggingStats {
if err == io.EOF && c.cfg.IsLoggingStats {
// This httpTransfer is done, we're checking it this way so we can also
// catch httpTransfers where the caller forgets to Close() the Body.
if c.response != nil {
@ -296,8 +301,8 @@ func (c *CountingReadCloser) Read(b []byte) (int, error) {
// LogHttpStats is intended to be called after all HTTP operations for the
// commmand have finished. It dumps k/v logs, one line per httpTransfer into
// a log file with the current timestamp.
func LogHttpStats() {
if !config.Config.IsLoggingStats {
func LogHttpStats(cfg *config.Configuration) {
if !cfg.IsLoggingStats {
return
}
@ -307,7 +312,7 @@ func LogHttpStats() {
return
}
fmt.Fprintf(file, "concurrent=%d batch=%v time=%d version=%s\n", config.Config.ConcurrentTransfers(), config.Config.BatchTransfer(), time.Now().Unix(), config.Version)
fmt.Fprintf(file, "concurrent=%d batch=%v time=%d version=%s\n", cfg.ConcurrentTransfers(), cfg.BatchTransfer(), time.Now().Unix(), config.Version)
for key, responses := range httpTransferBuckets {
for _, response := range responses {

27
httputil/ntlm.go
View File

@ -24,7 +24,7 @@ func ntlmClientSession(c *config.Configuration, creds auth.Creds) (ntlm.ClientSe
splits := strings.Split(creds["username"], "\\")
if len(splits) != 2 {
errorMessage := fmt.Sprintf("Your user name must be of the form DOMAIN\\user. It is currently %s", creds["username"], "string")
errorMessage := fmt.Sprintf("Your user name must be of the form DOMAIN\\user. It is currently %s", creds["username"])
return nil, errors.New(errorMessage)
}
@ -39,21 +39,20 @@ func ntlmClientSession(c *config.Configuration, creds auth.Creds) (ntlm.ClientSe
return session, nil
}
func doNTLMRequest(request *http.Request, retry bool) (*http.Response, error) {
func doNTLMRequest(cfg *config.Configuration, request *http.Request, retry bool) (*http.Response, error) {
handReq, err := cloneRequest(request)
if err != nil {
return nil, err
}
res, err := NewHttpClient(config.Config, handReq.Host).Do(handReq)
res, err := NewHttpClient(cfg, handReq.Host).Do(handReq)
if err != nil && res == nil {
return nil, err
}
//If the status is 401 then we need to re-authenticate, otherwise it was successful
if res.StatusCode == 401 {
creds, err := auth.GetCreds(config.Config, request)
creds, err := auth.GetCreds(cfg, request)
if err != nil {
return nil, err
}
@ -63,7 +62,7 @@ func doNTLMRequest(request *http.Request, retry bool) (*http.Response, error) {
return nil, err
}
challengeMessage, err := negotiate(negotiateReq, ntlmNegotiateMessage)
challengeMessage, err := negotiate(cfg, negotiateReq, ntlmNegotiateMessage)
if err != nil {
return nil, err
}
@ -73,26 +72,26 @@ func doNTLMRequest(request *http.Request, retry bool) (*http.Response, error) {
return nil, err
}
res, err := challenge(challengeReq, challengeMessage, creds)
res, err := challenge(cfg, challengeReq, challengeMessage, creds)
if err != nil {
return nil, err
}
//If the status is 401 then we need to re-authenticate
if res.StatusCode == 401 && retry == true {
return doNTLMRequest(challengeReq, false)
return doNTLMRequest(cfg, challengeReq, false)
}
auth.SaveCredentials(config.Config, creds, res)
auth.SaveCredentials(cfg, creds, res)
return res, nil
}
return res, nil
}
func negotiate(request *http.Request, message string) ([]byte, error) {
func negotiate(cfg *config.Configuration, request *http.Request, message string) ([]byte, error) {
request.Header.Add("Authorization", message)
res, err := NewHttpClient(config.Config, request.Host).Do(request)
res, err := NewHttpClient(cfg, request.Host).Do(request)
if res == nil && err != nil {
return nil, err
@ -109,13 +108,13 @@ func negotiate(request *http.Request, message string) ([]byte, error) {
return ret, nil
}
func challenge(request *http.Request, challengeBytes []byte, creds auth.Creds) (*http.Response, error) {
func challenge(cfg *config.Configuration, request *http.Request, challengeBytes []byte, creds auth.Creds) (*http.Response, error) {
challenge, err := ntlm.ParseChallengeMessage(challengeBytes)
if err != nil {
return nil, err
}
session, err := ntlmClientSession(config.Config, creds)
session, err := ntlmClientSession(cfg, creds)
if err != nil {
return nil, err
}
@ -128,7 +127,7 @@ func challenge(request *http.Request, challengeBytes []byte, creds auth.Creds) (
authMsg := base64.StdEncoding.EncodeToString(authenticate.Bytes())
request.Header.Add("Authorization", "NTLM "+authMsg)
return NewHttpClient(config.Config, request.Host).Do(request)
return NewHttpClient(cfg, request.Host).Do(request)
}
func parseChallengeResponse(response *http.Response) ([]byte, error) {

22
httputil/ntlm_test.go
View File

@ -14,34 +14,22 @@ import (
)
func TestNtlmClientSession(t *testing.T) {
//Make sure to clear ntlmSession so test order doesn't matter.
config.Config.NtlmSession = nil
cfg := config.New()
creds := auth.Creds{"username": "MOOSEDOMAIN\\canadian", "password": "MooseAntlersYeah"}
_, err := ntlmClientSession(config.Config, creds)
_, err := ntlmClientSession(cfg, creds)
assert.Nil(t, err)
//The second call should ignore creds and give the session we just created.
badCreds := auth.Creds{"username": "badusername", "password": "MooseAntlersYeah"}
_, err = ntlmClientSession(config.Config, badCreds)
_, err = ntlmClientSession(cfg, badCreds)
assert.Nil(t, err)
//clean up
config.Config.NtlmSession = nil
}
func TestNtlmClientSessionBadCreds(t *testing.T) {
//Make sure to clear ntlmSession so test order doesn't matter.
config.Config.NtlmSession = nil
cfg := config.New()
creds := auth.Creds{"username": "badusername", "password": "MooseAntlersYeah"}
_, err := ntlmClientSession(config.Config, creds)
_, err := ntlmClientSession(cfg, creds)
assert.NotNil(t, err)
//clean up
config.Config.NtlmSession = nil
}
func TestNtlmCloneRequest(t *testing.T) {

37
httputil/request.go
View File

@ -43,16 +43,16 @@ func (e *ClientError) Error() string {
}
// Internal http request management
func doHttpRequest(req *http.Request, creds auth.Creds) (*http.Response, error) {
func doHttpRequest(cfg *config.Configuration, req *http.Request, creds auth.Creds) (*http.Response, error) {
var (
res *http.Response
err error
)
if config.Config.NtlmAccess(auth.GetOperationForRequest(req)) {
res, err = doNTLMRequest(req, true)
if cfg.NtlmAccess(auth.GetOperationForRequest(req)) {
res, err = doNTLMRequest(cfg, req, true)
} else {
res, err = NewHttpClient(config.Config, req.Host).Do(req)
res, err = NewHttpClient(cfg, req.Host).Do(req)
}
if res == nil {
@ -66,20 +66,20 @@ func doHttpRequest(req *http.Request, creds auth.Creds) (*http.Response, error)
if err != nil {
if errutil.IsAuthError(err) {
SetAuthType(req, res)
doHttpRequest(req, creds)
SetAuthType(cfg, req, res)
doHttpRequest(cfg, req, creds)
} else {
err = errutil.Error(err)
}
} else {
err = handleResponse(res, creds)
err = handleResponse(cfg, res, creds)
}
if err != nil {
if res != nil {
SetErrorResponseContext(err, res)
SetErrorResponseContext(cfg, err, res)
} else {
setErrorRequestContext(err, req)
setErrorRequestContext(cfg, err, req)
}
}
@ -87,31 +87,31 @@ func doHttpRequest(req *http.Request, creds auth.Creds) (*http.Response, error)
}
// DoHttpRequest performs a single HTTP request
func DoHttpRequest(req *http.Request, useCreds bool) (*http.Response, error) {
func DoHttpRequest(cfg *config.Configuration, req *http.Request, useCreds bool) (*http.Response, error) {
var creds auth.Creds
if useCreds {
c, err := auth.GetCreds(config.Config, req)
c, err := auth.GetCreds(cfg, req)
if err != nil {
return nil, err
}
creds = c
}
return doHttpRequest(req, creds)
return doHttpRequest(cfg, req, creds)
}
// DoHttpRequestWithRedirects runs a HTTP request and responds to redirects
func DoHttpRequestWithRedirects(req *http.Request, via []*http.Request, useCreds bool) (*http.Response, error) {
func DoHttpRequestWithRedirects(cfg *config.Configuration, req *http.Request, via []*http.Request, useCreds bool) (*http.Response, error) {
var creds auth.Creds
if useCreds {
c, err := auth.GetCreds(config.Config, req)
c, err := auth.GetCreds(cfg, req)
if err != nil {
return nil, err
}
creds = c
}
res, err := doHttpRequest(req, creds)
res, err := doHttpRequest(cfg, req, creds)
if err != nil {
return res, err
}
@ -152,7 +152,7 @@ func DoHttpRequestWithRedirects(req *http.Request, via []*http.Request, useCreds
return res, errutil.Errorf(err, err.Error())
}
return DoHttpRequestWithRedirects(redirectedReq, via, useCreds)
return DoHttpRequestWithRedirects(cfg, redirectedReq, via, useCreds)
}
return res, nil
@ -174,15 +174,14 @@ func NewHttpRequest(method, rawurl string, header map[string]string) (*http.Requ
return req, nil
}
func SetAuthType(req *http.Request, res *http.Response) {
func SetAuthType(cfg *config.Configuration, req *http.Request, res *http.Response) {
authType := GetAuthType(res)
operation := auth.GetOperationForRequest(req)
config.Config.SetAccess(operation, authType)
cfg.SetAccess(operation, authType)
tracerx.Printf("api: http response indicates %q authentication. Resubmitting...", authType)
}
func GetAuthType(res *http.Response) string {
for _, headerName := range authenticateHeaders {
for _, auth := range res.Header[headerName] {

10
httputil/request_error_test.go
View File

@ -9,19 +9,22 @@ import (
"net/url"
"testing"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
)
func TestSuccessStatus(t *testing.T) {
cfg := config.New()
for _, status := range []int{200, 201, 202} {
res := &http.Response{StatusCode: status}
if err := handleResponse(res, nil); err != nil {
if err := handleResponse(cfg, res, nil); err != nil {
t.Errorf("Unexpected error for HTTP %d: %s", status, err.Error())
}
}
}
func TestErrorStatusWithCustomMessage(t *testing.T) {
cfg := config.New()
u, err := url.Parse("https://lfs-server.com/objects/oid")
if err != nil {
t.Fatal(err)
@ -61,7 +64,7 @@ func TestErrorStatusWithCustomMessage(t *testing.T) {
}
res.Header.Set("Content-Type", "application/vnd.git-lfs+json; charset=utf-8")
err = handleResponse(res, nil)
err = handleResponse(cfg, res, nil)
if err == nil {
t.Errorf("No error from HTTP %d", status)
continue
@ -81,6 +84,7 @@ func TestErrorStatusWithCustomMessage(t *testing.T) {
}
func TestErrorStatusWithDefaultMessage(t *testing.T) {
cfg := config.New()
rawurl := "https://lfs-server.com/objects/oid"
u, err := url.Parse(rawurl)
if err != nil {
@ -123,7 +127,7 @@ func TestErrorStatusWithDefaultMessage(t *testing.T) {
// purposely wrong content type so it falls back to default
res.Header.Set("Content-Type", "application/vnd.git-lfs+json2")
err = handleResponse(res, nil)
err = handleResponse(cfg, res, nil)
if err == nil {
t.Errorf("No error from HTTP %d", status)
continue

12
httputil/response.go
View File

@ -56,8 +56,8 @@ func GetDefaultError(code int) string {
}
// Check the response from a HTTP request for problems
func handleResponse(res *http.Response, creds auth.Creds) error {
auth.SaveCredentials(config.Config, creds, res)
func handleResponse(cfg *config.Configuration, res *http.Response, creds auth.Creds) error {
auth.SaveCredentials(cfg, creds, res)
if res.StatusCode < 400 {
return nil
@ -103,14 +103,14 @@ func defaultError(res *http.Response) error {
return errutil.Error(fmt.Errorf(msgFmt, res.Request.URL))
}
func SetErrorResponseContext(err error, res *http.Response) {
func SetErrorResponseContext(cfg *config.Configuration, err error, res *http.Response) {
errutil.ErrorSetContext(err, "Status", res.Status)
setErrorHeaderContext(err, "Request", res.Header)
setErrorRequestContext(err, res.Request)
setErrorRequestContext(cfg, err, res.Request)
}
func setErrorRequestContext(err error, req *http.Request) {
errutil.ErrorSetContext(err, "Endpoint", config.Config.Endpoint(auth.GetOperationForRequest(req)).Url)
func setErrorRequestContext(cfg *config.Configuration, err error, req *http.Request) {
errutil.ErrorSetContext(err, "Endpoint", cfg.Endpoint(auth.GetOperationForRequest(req)).Url)
errutil.ErrorSetContext(err, "URL", TraceHttpReq(req))
setErrorHeaderContext(err, "Response", req.Header)
}

7
test/cmd/lfstest-customadapter.go
View File

@ -13,11 +13,14 @@ import (
"strings"
"time"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/httputil"
"github.com/github/git-lfs/progress"
"github.com/github/git-lfs/tools"
)
var cfg = config.New()
// This test custom adapter just acts as a bridge for uploads/downloads
// in order to demonstrate & test the custom transfer adapter protocols
// All we actually do is relay the requests back to the normal storage URLs
@ -103,7 +106,7 @@ func performDownload(oid string, size int64, a *action, writer, errWriter *bufio
sendTransferError(oid, 2, err.Error(), writer, errWriter)
return
}
res, err := httputil.DoHttpRequest(req, true)
res, err := httputil.DoHttpRequest(cfg, req, true)
if err != nil {
sendTransferError(oid, res.StatusCode, err.Error(), writer, errWriter)
return
@ -184,7 +187,7 @@ func performUpload(oid string, size int64, a *action, fromPath string, writer, e
req.Body = ioutil.NopCloser(reader)
res, err := httputil.DoHttpRequest(req, true)
res, err := httputil.DoHttpRequest(cfg, req, true)
if err != nil {
sendTransferError(oid, res.StatusCode, fmt.Sprintf("Error uploading data for %s: %v", oid, err), writer, errWriter)
return

6
transfer/basic_download.go
View File

@ -10,6 +10,7 @@ import (
"regexp"
"strconv"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/httputil"
"github.com/github/git-lfs/localstorage"
@ -85,7 +86,6 @@ func (a *basicDownloadAdapter) downloadFilename(t *Transfer) string {
// download starts or resumes and download. Always closes dlFile if non-nil
func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback, authOkFunc func(), dlFile *os.File, fromByte int64, hash hash.Hash) error {
if dlFile != nil {
// ensure we always close dlFile. Note that this does not conflict with the
// early close below, as close is idempotent.
@ -110,7 +110,7 @@ func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", fromByte, t.Object.Size-1))
}
res, err := httputil.DoHttpRequest(req, true)
res, err := httputil.DoHttpRequest(config.Config, req, true)
if err != nil {
// Special-case status code 416 () - fall back
if fromByte > 0 && dlFile != nil && res.StatusCode == 416 {
@ -121,7 +121,7 @@ func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback
}
return errutil.NewRetriableError(err)
}
httputil.LogTransfer("lfs.data.download", res)
httputil.LogTransfer(config.Config, "lfs.data.download", res)
defer res.Body.Close()
// Range request must return 206 & content range to confirm

5
transfer/basic_upload.go
View File

@ -9,6 +9,7 @@ import (
"strconv"
"github.com/github/git-lfs/api"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/httputil"
"github.com/github/git-lfs/progress"
@ -96,11 +97,11 @@ func (a *basicUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb Transfe
req.Body = ioutil.NopCloser(reader)
res, err := httputil.DoHttpRequest(req, true)
res, err := httputil.DoHttpRequest(config.Config, req, true)
if err != nil {
return errutil.NewRetriableError(err)
}
httputil.LogTransfer("lfs.data.upload", res)
httputil.LogTransfer(config.Config, "lfs.data.upload", res)
// A status code of 403 likely means that an authentication token for the
// upload has expired. This can be safely retried.

7
transfer/tus_upload.go
View File

@ -8,6 +8,7 @@ import (
"strconv"
"github.com/github/git-lfs/api"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/httputil"
"github.com/github/git-lfs/progress"
@ -52,7 +53,7 @@ func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferP
return err
}
req.Header.Set("Tus-Resumable", TusVersion)
res, err := httputil.DoHttpRequest(req, false)
res, err := httputil.DoHttpRequest(config.Config, req, false)
if err != nil {
return errutil.NewRetriableError(err)
}
@ -133,11 +134,11 @@ func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferP
req.Body = ioutil.NopCloser(reader)
res, err = httputil.DoHttpRequest(req, false)
res, err = httputil.DoHttpRequest(config.Config, req, false)
if err != nil {
return errutil.NewRetriableError(err)
}
httputil.LogTransfer("lfs.data.upload", res)
httputil.LogTransfer(config.Config, "lfs.data.upload", res)
// A status code of 403 likely means that an authentication token for the
// upload has expired. This can be safely retried.