diff --git a/lfsapi/certs.go b/lfsapi/certs.go new file mode 100644 index 00000000..5bb3b71a --- /dev/null +++ b/lfsapi/certs.go @@ -0,0 +1,132 @@ +package lfsapi + +import ( + "crypto/x509" + "fmt" + "io/ioutil" + "path/filepath" + + "github.com/rubyist/tracerx" +) + +// isCertVerificationDisabledForHost returns whether SSL certificate verification +// has been disabled for the given host, or globally +func isCertVerificationDisabledForHost(c *Client, host string) bool { + hostSslVerify, _ := c.gitEnv.Get(fmt.Sprintf("http.https://%v/.sslverify", host)) + if hostSslVerify == "false" { + return true + } + + return c.SkipSSLVerify +} + +// getRootCAsForHost returns a certificate pool for that specific host (which may +// be "host:port" loaded from either the gitconfig or from a platform-specific +// source which is not included by default in the golang certificate search) +// May return nil if it doesn't have anything to add, in which case the default +// RootCAs will be used if passed to TLSClientConfig.RootCAs +func getRootCAsForHost(c *Client, host string) *x509.CertPool { + // don't init pool, want to return nil not empty if none found; init only on successful add cert + var pool *x509.CertPool + + // gitconfig first + pool = appendRootCAsForHostFromGitconfig(c.osEnv, c.gitEnv, pool, host) + // Platform specific + return appendRootCAsForHostFromPlatform(pool, host) +} + +func appendRootCAsForHostFromGitconfig(osEnv env, gitEnv env, pool *x509.CertPool, host string) *x509.CertPool { + // Accumulate certs from all these locations: + + // GIT_SSL_CAINFO first + if cafile, _ := osEnv.Get("GIT_SSL_CAINFO"); len(cafile) > 0 { + return appendCertsFromFile(pool, cafile) + } + // http./.sslcainfo or http..sslcainfo + // we know we have simply "host" or "host:port" + hostKeyWithSlash := fmt.Sprintf("http.https://%v/.sslcainfo", host) + if cafile, ok := gitEnv.Get(hostKeyWithSlash); ok { + return appendCertsFromFile(pool, cafile) + } + hostKeyWithoutSlash := fmt.Sprintf("http.https://%v.sslcainfo", host) + if cafile, ok := gitEnv.Get(hostKeyWithoutSlash); ok { + return appendCertsFromFile(pool, cafile) + } + // http.sslcainfo + if cafile, ok := gitEnv.Get("http.sslcainfo"); ok { + return appendCertsFromFile(pool, cafile) + } + // GIT_SSL_CAPATH + if cadir, _ := osEnv.Get("GIT_SSL_CAPATH"); len(cadir) > 0 { + return appendCertsFromFilesInDir(pool, cadir) + } + // http.sslcapath + if cadir, ok := gitEnv.Get("http.sslcapath"); ok { + return appendCertsFromFilesInDir(pool, cadir) + } + + return pool +} + +func appendCertsFromFilesInDir(pool *x509.CertPool, dir string) *x509.CertPool { + files, err := ioutil.ReadDir(dir) + if err != nil { + tracerx.Printf("Error reading cert dir %q: %v", dir, err) + return pool + } + for _, f := range files { + pool = appendCertsFromFile(pool, filepath.Join(dir, f.Name())) + } + return pool +} + +func appendCertsFromFile(pool *x509.CertPool, filename string) *x509.CertPool { + data, err := ioutil.ReadFile(filename) + if err != nil { + tracerx.Printf("Error reading cert file %q: %v", filename, err) + return pool + } + // Firstly, try parsing as binary certificate + if certs, err := x509.ParseCertificates(data); err == nil { + return appendCerts(pool, certs) + } + // If not binary certs, try PEM data + return appendCertsFromPEMData(pool, data) +} + +func appendCerts(pool *x509.CertPool, certs []*x509.Certificate) *x509.CertPool { + if len(certs) == 0 { + // important to return unmodified (may be nil) + return pool + } + + if pool == nil { + pool = x509.NewCertPool() + } + + for _, cert := range certs { + pool.AddCert(cert) + } + + return pool +} + +func appendCertsFromPEMData(pool *x509.CertPool, data []byte) *x509.CertPool { + if len(data) == 0 { + return pool + } + + // Bit of a dance, need to ensure if AppendCertsFromPEM fails we still return + // nil and not an empty pool, so system roots still get used + var ret *x509.CertPool + if pool == nil { + ret = x509.NewCertPool() + } else { + ret = pool + } + if !ret.AppendCertsFromPEM(data) { + // Return unmodified input pool (may be nil, do not replace with empty) + return pool + } + return ret +} diff --git a/lfsapi/certs_darwin.go b/lfsapi/certs_darwin.go new file mode 100644 index 00000000..513678fc --- /dev/null +++ b/lfsapi/certs_darwin.go @@ -0,0 +1,64 @@ +package lfsapi + +import ( + "crypto/x509" + "regexp" + "strings" + + "github.com/git-lfs/git-lfs/subprocess" + "github.com/rubyist/tracerx" +) + +func appendRootCAsForHostFromPlatform(pool *x509.CertPool, host string) *x509.CertPool { + // Go loads only the system root certificates by default + // see https://github.com/golang/go/blob/master/src/crypto/x509/root_darwin.go + // We want to load certs configured in the System keychain too, this is separate + // from the system root certificates. It's also where other tools such as + // browsers (e.g. Chrome) will load custom trusted certs from. They often + // don't load certs from the login keychain so that's not included here + // either, for consistency. + + // find system.keychain for user-added certs (don't assume location) + cmd := subprocess.ExecCommand("/usr/bin/security", "list-keychains") + kcout, err := cmd.Output() + if err != nil { + tracerx.Printf("Error listing keychains: %v", err) + return nil + } + + var systemKeychain string + keychains := strings.Split(string(kcout), "\n") + for _, keychain := range keychains { + lc := strings.ToLower(keychain) + if !strings.Contains(lc, "/system.keychain") { + continue + } + systemKeychain = strings.Trim(keychain, " \t\"") + break + } + + if len(systemKeychain) == 0 { + return nil + } + + pool = appendRootCAsFromKeychain(pool, host, systemKeychain) + + // Also check host without port + portreg := regexp.MustCompile(`([^:]+):\d+`) + if match := portreg.FindStringSubmatch(host); match != nil { + hostwithoutport := match[1] + pool = appendRootCAsFromKeychain(pool, hostwithoutport, systemKeychain) + } + + return pool +} + +func appendRootCAsFromKeychain(pool *x509.CertPool, name, keychain string) *x509.CertPool { + cmd := subprocess.ExecCommand("/usr/bin/security", "find-certificate", "-a", "-p", "-c", name, keychain) + data, err := cmd.Output() + if err != nil { + tracerx.Printf("Error reading keychain %q: %v", keychain, err) + return pool + } + return appendCertsFromPEMData(pool, data) +} diff --git a/lfsapi/certs_freebsd.go b/lfsapi/certs_freebsd.go new file mode 100644 index 00000000..989f4ee9 --- /dev/null +++ b/lfsapi/certs_freebsd.go @@ -0,0 +1,8 @@ +package lfsapi + +import "crypto/x509" + +func appendRootCAsForHostFromPlatform(pool *x509.CertPool, host string) *x509.CertPool { + // Do nothing, use golang default + return pool +} diff --git a/lfsapi/certs_linux.go b/lfsapi/certs_linux.go new file mode 100644 index 00000000..989f4ee9 --- /dev/null +++ b/lfsapi/certs_linux.go @@ -0,0 +1,8 @@ +package lfsapi + +import "crypto/x509" + +func appendRootCAsForHostFromPlatform(pool *x509.CertPool, host string) *x509.CertPool { + // Do nothing, use golang default + return pool +} diff --git a/lfsapi/certs_openbsd.go b/lfsapi/certs_openbsd.go new file mode 100644 index 00000000..989f4ee9 --- /dev/null +++ b/lfsapi/certs_openbsd.go @@ -0,0 +1,8 @@ +package lfsapi + +import "crypto/x509" + +func appendRootCAsForHostFromPlatform(pool *x509.CertPool, host string) *x509.CertPool { + // Do nothing, use golang default + return pool +} diff --git a/lfsapi/certs_test.go b/lfsapi/certs_test.go new file mode 100644 index 00000000..0eb67a5f --- /dev/null +++ b/lfsapi/certs_test.go @@ -0,0 +1,230 @@ +package lfsapi + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +var testCert = `-----BEGIN CERTIFICATE----- +MIIDyjCCArKgAwIBAgIJAMi9TouXnW+ZMA0GCSqGSIb3DQEBBQUAMEwxCzAJBgNV +BAYTAlVTMRMwEQYDVQQIEwpTb21lLVN0YXRlMRAwDgYDVQQKEwdnaXQtbGZzMRYw +FAYDVQQDEw1naXQtbGZzLmxvY2FsMB4XDTE2MDMwOTEwNTk1NFoXDTI2MDMwNzEw +NTk1NFowTDELMAkGA1UEBhMCVVMxEzARBgNVBAgTClNvbWUtU3RhdGUxEDAOBgNV +BAoTB2dpdC1sZnMxFjAUBgNVBAMTDWdpdC1sZnMubG9jYWwwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCXmsI2w44nOsP7n3kL1Lz04U5FMZRErBSXLOE+ +dpd4tMpgrjOncJPD9NapHabsVIOnuVvMDuBbWYwU9PwbN4tjQzch8DRxBju6fCp/ +Pm+QF6p2Ga+NuSHWoVfNFuF2776aF9gSLC0rFnBekD3HCz+h6I5HFgHBvRjeVyAs +PRw471Y28Je609SoYugxaQNzRvahP0Qf43tE74/WN3FTGXy1+iU+uXpfp8KxnsuB +gfj+Wi6mPt8Q2utcA1j82dJ0K8ZbHSbllzmI+N/UuRLsbTUEdeFWYdZ0AlZNd/Vc +PlOSeoExwvOHIuUasT/cLIrEkdXNud2QLg2GpsB6fJi3NEUhAgMBAAGjga4wgasw +HQYDVR0OBBYEFC8oVPRQbekTwfkntgdL7PADXNDbMHwGA1UdIwR1MHOAFC8oVPRQ +bekTwfkntgdL7PADXNDboVCkTjBMMQswCQYDVQQGEwJVUzETMBEGA1UECBMKU29t +ZS1TdGF0ZTEQMA4GA1UEChMHZ2l0LWxmczEWMBQGA1UEAxMNZ2l0LWxmcy5sb2Nh +bIIJAMi9TouXnW+ZMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBACIl +/CBLIhC3drrYme4cGArhWyXIyRpMoy9Z+9Dru8rSuOr/RXR6sbYhlE1iMGg4GsP8 +4Cj7aIct6Vb9NFv5bGNyFJAmDesm3SZlEcWxU3YBzNPiJXGiUpQHCkp0BH+gvsXc +tb58XoiDZPVqrl0jNfX/nHpHR9c3DaI3Tjx0F/No0ZM6mLQ1cNMikFyEWQ4U0zmW +LvV+vvKuOixRqbcVnB5iTxqMwFG0X3tUql0cftGBgoCoR1+FSBOs0EXLODCck6ql +aW6vZwkA+ccj/pDTx8LBe2lnpatrFeIt6znAUJW3G8r6SFHKVBWHwmESZS4kxhjx +NpW5Hh0w4/5iIetCkJ0= +-----END CERTIFICATE-----` + +var sslCAInfoConfigHostNames = []string{ + "git-lfs.local", + "git-lfs.local/", +} +var sslCAInfoMatchedHostTests = []struct { + hostName string + shouldMatch bool +}{ + {"git-lfs.local", true}, + {"git-lfs.local:8443", false}, + {"wronghost.com", false}, +} + +func TestCertFromSSLCAInfoConfig(t *testing.T) { + tempfile, err := ioutil.TempFile("", "testcert") + assert.Nil(t, err, "Error creating temp cert file") + defer os.Remove(tempfile.Name()) + + _, err = tempfile.WriteString(testCert) + assert.Nil(t, err, "Error writing temp cert file") + tempfile.Close() + + // Test http..sslcainfo + for _, hostName := range sslCAInfoConfigHostNames { + hostKey := fmt.Sprintf("http.https://%v.sslcainfo", hostName) + c, err := NewClient(nil, testEnv(map[string]string{ + hostKey: tempfile.Name(), + })) + assert.Nil(t, err) + + for _, matchedHostTest := range sslCAInfoMatchedHostTests { + pool := getRootCAsForHost(c, matchedHostTest.hostName) + + var shouldOrShouldnt string + if matchedHostTest.shouldMatch { + shouldOrShouldnt = "should" + } else { + shouldOrShouldnt = "should not" + } + + assert.Equal(t, matchedHostTest.shouldMatch, pool != nil, + "Cert lookup for \"%v\" %v have succeeded with \"%v\"", + matchedHostTest.hostName, shouldOrShouldnt, hostKey) + } + } + + // Test http.sslcainfo + c, err := NewClient(nil, testEnv(map[string]string{ + "http.sslcainfo": tempfile.Name(), + })) + assert.Nil(t, err) + + // Should match any host at all + for _, matchedHostTest := range sslCAInfoMatchedHostTests { + pool := getRootCAsForHost(c, matchedHostTest.hostName) + assert.NotNil(t, pool) + } +} + +func TestCertFromSSLCAInfoEnv(t *testing.T) { + tempfile, err := ioutil.TempFile("", "testcert") + assert.Nil(t, err, "Error creating temp cert file") + defer os.Remove(tempfile.Name()) + + _, err = tempfile.WriteString(testCert) + assert.Nil(t, err, "Error writing temp cert file") + tempfile.Close() + + c, err := NewClient(testEnv(map[string]string{ + "GIT_SSL_CAINFO": tempfile.Name(), + }), nil) + assert.Nil(t, err) + + // Should match any host at all + for _, matchedHostTest := range sslCAInfoMatchedHostTests { + pool := getRootCAsForHost(c, matchedHostTest.hostName) + assert.NotNil(t, pool) + } +} + +func TestCertFromSSLCAPathConfig(t *testing.T) { + tempdir, err := ioutil.TempDir("", "testcertdir") + assert.Nil(t, err, "Error creating temp cert dir") + defer os.RemoveAll(tempdir) + + err = ioutil.WriteFile(filepath.Join(tempdir, "cert1.pem"), []byte(testCert), 0644) + assert.Nil(t, err, "Error creating cert file") + + c, err := NewClient(nil, testEnv(map[string]string{ + "http.sslcapath": tempdir, + })) + + assert.Nil(t, err) + + // Should match any host at all + for _, matchedHostTest := range sslCAInfoMatchedHostTests { + pool := getRootCAsForHost(c, matchedHostTest.hostName) + assert.NotNil(t, pool) + } +} + +func TestCertFromSSLCAPathEnv(t *testing.T) { + tempdir, err := ioutil.TempDir("", "testcertdir") + assert.Nil(t, err, "Error creating temp cert dir") + defer os.RemoveAll(tempdir) + + err = ioutil.WriteFile(filepath.Join(tempdir, "cert1.pem"), []byte(testCert), 0644) + assert.Nil(t, err, "Error creating cert file") + + c, err := NewClient(testEnv(map[string]string{ + "GIT_SSL_CAPATH": tempdir, + }), nil) + assert.Nil(t, err) + + // Should match any host at all + for _, matchedHostTest := range sslCAInfoMatchedHostTests { + pool := getRootCAsForHost(c, matchedHostTest.hostName) + assert.NotNil(t, pool) + } +} + +func TestCertVerifyDisabledGlobalEnv(t *testing.T) { + empty := &Client{} + httpClient := empty.httpClient("anyhost.com") + tr, ok := httpClient.Transport.(*http.Transport) + if assert.True(t, ok) { + assert.False(t, tr.TLSClientConfig.InsecureSkipVerify) + } + + c, err := NewClient(testEnv(map[string]string{ + "GIT_SSL_NO_VERIFY": "1", + }), nil) + + assert.Nil(t, err) + + httpClient = c.httpClient("anyhost.com") + tr, ok = httpClient.Transport.(*http.Transport) + if assert.True(t, ok) { + assert.True(t, tr.TLSClientConfig.InsecureSkipVerify) + } +} + +func TestCertVerifyDisabledGlobalConfig(t *testing.T) { + def := &Client{} + httpClient := def.httpClient("anyhost.com") + tr, ok := httpClient.Transport.(*http.Transport) + if assert.True(t, ok) { + assert.False(t, tr.TLSClientConfig.InsecureSkipVerify) + } + + c, err := NewClient(nil, testEnv(map[string]string{ + "http.sslverify": "false", + })) + assert.Nil(t, err) + + httpClient = c.httpClient("anyhost.com") + tr, ok = httpClient.Transport.(*http.Transport) + if assert.True(t, ok) { + assert.True(t, tr.TLSClientConfig.InsecureSkipVerify) + } +} + +func TestCertVerifyDisabledHostConfig(t *testing.T) { + def := &Client{} + httpClient := def.httpClient("specifichost.com") + tr, ok := httpClient.Transport.(*http.Transport) + if assert.True(t, ok) { + assert.False(t, tr.TLSClientConfig.InsecureSkipVerify) + } + + httpClient = def.httpClient("otherhost.com") + tr, ok = httpClient.Transport.(*http.Transport) + if assert.True(t, ok) { + assert.False(t, tr.TLSClientConfig.InsecureSkipVerify) + } + + c, err := NewClient(nil, testEnv(map[string]string{ + "http.https://specifichost.com/.sslverify": "false", + })) + assert.Nil(t, err) + + httpClient = c.httpClient("specifichost.com") + tr, ok = httpClient.Transport.(*http.Transport) + if assert.True(t, ok) { + assert.True(t, tr.TLSClientConfig.InsecureSkipVerify) + } + + httpClient = c.httpClient("otherhost.com") + tr, ok = httpClient.Transport.(*http.Transport) + if assert.True(t, ok) { + assert.False(t, tr.TLSClientConfig.InsecureSkipVerify) + } +} diff --git a/lfsapi/certs_windows.go b/lfsapi/certs_windows.go new file mode 100644 index 00000000..d2f8a890 --- /dev/null +++ b/lfsapi/certs_windows.go @@ -0,0 +1,8 @@ +package lfsapi + +import "crypto/x509" + +func appendRootCAsForHostFromPlatform(pool *x509.CertPool, host string) *x509.CertPool { + // golang already supports Windows Certificate Store for self-signed certs + return pool +} diff --git a/lfsapi/client_test.go b/lfsapi/client_test.go index bb6f770b..bd66f1d2 100644 --- a/lfsapi/client_test.go +++ b/lfsapi/client_test.go @@ -21,3 +21,51 @@ func TestNewClient(t *testing.T) { assert.Equal(t, 153, c.TLSTimeout) assert.Equal(t, 154, c.ConcurrentTransfers) } + +func TestNewClientWithGitSSLVerify(t *testing.T) { + c, err := NewClient(nil, nil) + assert.Nil(t, err) + assert.False(t, c.SkipSSLVerify) + + for _, value := range []string{"true", "1", "t"} { + c, err = NewClient(testEnv(map[string]string{}), testEnv(map[string]string{ + "http.sslverify": value, + })) + t.Logf("http.sslverify: %q", value) + assert.Nil(t, err) + assert.False(t, c.SkipSSLVerify) + } + + for _, value := range []string{"false", "0", "f"} { + c, err = NewClient(testEnv(map[string]string{}), testEnv(map[string]string{ + "http.sslverify": value, + })) + t.Logf("http.sslverify: %q", value) + assert.Nil(t, err) + assert.True(t, c.SkipSSLVerify) + } +} + +func TestNewClientWithOSSSLVerify(t *testing.T) { + c, err := NewClient(nil, nil) + assert.Nil(t, err) + assert.False(t, c.SkipSSLVerify) + + for _, value := range []string{"false", "0", "f"} { + c, err = NewClient(testEnv(map[string]string{ + "GIT_SSL_NO_VERIFY": value, + }), testEnv(map[string]string{})) + t.Logf("GIT_SSL_NO_VERIFY: %q", value) + assert.Nil(t, err) + assert.False(t, c.SkipSSLVerify) + } + + for _, value := range []string{"true", "1", "t"} { + c, err = NewClient(testEnv(map[string]string{ + "GIT_SSL_NO_VERIFY": value, + }), testEnv(map[string]string{})) + t.Logf("GIT_SSL_NO_VERIFY: %q", value) + assert.Nil(t, err) + assert.True(t, c.SkipSSLVerify) + } +} diff --git a/lfsapi/lfsapi.go b/lfsapi/lfsapi.go index d832217f..7cce9685 100644 --- a/lfsapi/lfsapi.go +++ b/lfsapi/lfsapi.go @@ -1,6 +1,7 @@ package lfsapi import ( + "crypto/tls" "encoding/json" "net" "net/http" @@ -30,9 +31,14 @@ type Client struct { HTTPSProxy string HTTPProxy string NoProxy string + SkipSSLVerify bool hostClients map[string]*http.Client clientMu sync.Mutex + + // only used for per-host ssl certs + gitEnv env + osEnv env } func NewClient(osEnv env, gitEnv env) (*Client, error) { @@ -61,9 +67,12 @@ func NewClient(osEnv env, gitEnv env) (*Client, error) { KeepaliveTimeout: gitEnv.Int("lfs.keepalive", 0), TLSTimeout: gitEnv.Int("lfs.tlstimeout", 0), ConcurrentTransfers: gitEnv.Int("lfs.concurrenttransfers", 0), + SkipSSLVerify: !gitEnv.Bool("http.sslverify", true) || osEnv.Bool("GIT_SSL_NO_VERIFY", false), HTTPSProxy: httpsProxy, HTTPProxy: httpProxy, NoProxy: noProxy, + gitEnv: gitEnv, + osEnv: osEnv, } return c, nil @@ -82,6 +91,14 @@ func (c *Client) httpClient(host string) *http.Client { c.clientMu.Lock() defer c.clientMu.Unlock() + if c.gitEnv == nil { + c.gitEnv = make(testEnv) + } + + if c.osEnv == nil { + c.osEnv = make(testEnv) + } + if c.hostClients == nil { c.hostClients = make(map[string]*http.Client) } @@ -120,6 +137,13 @@ func (c *Client) httpClient(host string) *http.Client { MaxIdleConnsPerHost: concurrentTransfers, } + tr.TLSClientConfig = &tls.Config{} + if isCertVerificationDisabledForHost(c, host) { + tr.TLSClientConfig.InsecureSkipVerify = true + } else { + tr.TLSClientConfig.RootCAs = getRootCAsForHost(c, host) + } + httpClient := &http.Client{ Transport: tr, }