lfsapi: add custom ssl cert support

lfsapi/certs.go

@@ -0,0 +1,132 @@
+package lfsapi
+
+import (
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/rubyist/tracerx"
+)
+
+// isCertVerificationDisabledForHost returns whether SSL certificate verification
+// has been disabled for the given host, or globally
+func isCertVerificationDisabledForHost(c *Client, host string) bool {
+	hostSslVerify, _ := c.gitEnv.Get(fmt.Sprintf("http.https://%v/.sslverify", host))
+	if hostSslVerify == "false" {
+		return true
+	}
+
+	return c.SkipSSLVerify
+}
+
+// getRootCAsForHost returns a certificate pool for that specific host (which may
+// be "host:port" loaded from either the gitconfig or from a platform-specific
+// source which is not included by default in the golang certificate search)
+// May return nil if it doesn't have anything to add, in which case the default
+// RootCAs will be used if passed to TLSClientConfig.RootCAs
+func getRootCAsForHost(c *Client, host string) *x509.CertPool {
+	// don't init pool, want to return nil not empty if none found; init only on successful add cert
+	var pool *x509.CertPool
+
+	// gitconfig first
+	pool = appendRootCAsForHostFromGitconfig(c.osEnv, c.gitEnv, pool, host)
+	// Platform specific
+	return appendRootCAsForHostFromPlatform(pool, host)
+}
+
+func appendRootCAsForHostFromGitconfig(osEnv env, gitEnv env, pool *x509.CertPool, host string) *x509.CertPool {
+	// Accumulate certs from all these locations:
+
+	// GIT_SSL_CAINFO first
+	if cafile, _ := osEnv.Get("GIT_SSL_CAINFO"); len(cafile) > 0 {
+		return appendCertsFromFile(pool, cafile)
+	}
+	// http.<url>/.sslcainfo or http.<url>.sslcainfo
+	// we know we have simply "host" or "host:port"
+	hostKeyWithSlash := fmt.Sprintf("http.https://%v/.sslcainfo", host)
+	if cafile, ok := gitEnv.Get(hostKeyWithSlash); ok {
+		return appendCertsFromFile(pool, cafile)
+	}
+	hostKeyWithoutSlash := fmt.Sprintf("http.https://%v.sslcainfo", host)
+	if cafile, ok := gitEnv.Get(hostKeyWithoutSlash); ok {
+		return appendCertsFromFile(pool, cafile)
+	}
+	// http.sslcainfo
+	if cafile, ok := gitEnv.Get("http.sslcainfo"); ok {
+		return appendCertsFromFile(pool, cafile)
+	}
+	// GIT_SSL_CAPATH
+	if cadir, _ := osEnv.Get("GIT_SSL_CAPATH"); len(cadir) > 0 {
+		return appendCertsFromFilesInDir(pool, cadir)
+	}
+	// http.sslcapath
+	if cadir, ok := gitEnv.Get("http.sslcapath"); ok {
+		return appendCertsFromFilesInDir(pool, cadir)
+	}
+
+	return pool
+}
+
+func appendCertsFromFilesInDir(pool *x509.CertPool, dir string) *x509.CertPool {
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		tracerx.Printf("Error reading cert dir %q: %v", dir, err)
+		return pool
+	}
+	for _, f := range files {
+		pool = appendCertsFromFile(pool, filepath.Join(dir, f.Name()))
+	}
+	return pool
+}
+
+func appendCertsFromFile(pool *x509.CertPool, filename string) *x509.CertPool {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		tracerx.Printf("Error reading cert file %q: %v", filename, err)
+		return pool
+	}
+	// Firstly, try parsing as binary certificate
+	if certs, err := x509.ParseCertificates(data); err == nil {
+		return appendCerts(pool, certs)
+	}
+	// If not binary certs, try PEM data
+	return appendCertsFromPEMData(pool, data)
+}
+
+func appendCerts(pool *x509.CertPool, certs []*x509.Certificate) *x509.CertPool {
+	if len(certs) == 0 {
+		// important to return unmodified (may be nil)
+		return pool
+	}
+
+	if pool == nil {
+		pool = x509.NewCertPool()
+	}
+
+	for _, cert := range certs {
+		pool.AddCert(cert)
+	}
+
+	return pool
+}
+
+func appendCertsFromPEMData(pool *x509.CertPool, data []byte) *x509.CertPool {
+	if len(data) == 0 {
+		return pool
+	}
+
+	// Bit of a dance, need to ensure if AppendCertsFromPEM fails we still return
+	// nil and not an empty pool, so system roots still get used
+	var ret *x509.CertPool
+	if pool == nil {
+		ret = x509.NewCertPool()
+	} else {
+		ret = pool
+	}
+	if !ret.AppendCertsFromPEM(data) {
+		// Return unmodified input pool (may be nil, do not replace with empty)
+		return pool
+	}
+	return ret
+}

lfsapi/certs_darwin.go

@@ -0,0 +1,64 @@
+package lfsapi
+
+import (
+	"crypto/x509"
+	"regexp"
+	"strings"
+
+	"github.com/git-lfs/git-lfs/subprocess"
+	"github.com/rubyist/tracerx"
+)
+
+func appendRootCAsForHostFromPlatform(pool *x509.CertPool, host string) *x509.CertPool {
+	// Go loads only the system root certificates by default
+	// see https://github.com/golang/go/blob/master/src/crypto/x509/root_darwin.go
+	// We want to load certs configured in the System keychain too, this is separate
+	// from the system root certificates. It's also where other tools such as
+	// browsers (e.g. Chrome) will load custom trusted certs from. They often
+	// don't load certs from the login keychain so that's not included here
+	// either, for consistency.
+
+	// find system.keychain for user-added certs (don't assume location)
+	cmd := subprocess.ExecCommand("/usr/bin/security", "list-keychains")
+	kcout, err := cmd.Output()
+	if err != nil {
+		tracerx.Printf("Error listing keychains: %v", err)
+		return nil
+	}
+
+	var systemKeychain string
+	keychains := strings.Split(string(kcout), "\n")
+	for _, keychain := range keychains {
+		lc := strings.ToLower(keychain)
+		if !strings.Contains(lc, "/system.keychain") {
+			continue
+		}
+		systemKeychain = strings.Trim(keychain, " \t\"")
+		break
+	}
+
+	if len(systemKeychain) == 0 {
+		return nil
+	}
+
+	pool = appendRootCAsFromKeychain(pool, host, systemKeychain)
+
+	// Also check host without port
+	portreg := regexp.MustCompile(`([^:]+):\d+`)
+	if match := portreg.FindStringSubmatch(host); match != nil {
+		hostwithoutport := match[1]
+		pool = appendRootCAsFromKeychain(pool, hostwithoutport, systemKeychain)
+	}
+
+	return pool
+}
+
+func appendRootCAsFromKeychain(pool *x509.CertPool, name, keychain string) *x509.CertPool {
+	cmd := subprocess.ExecCommand("/usr/bin/security", "find-certificate", "-a", "-p", "-c", name, keychain)
+	data, err := cmd.Output()
+	if err != nil {
+		tracerx.Printf("Error reading keychain %q: %v", keychain, err)
+		return pool
+	}
+	return appendCertsFromPEMData(pool, data)
+}

lfsapi/certs_freebsd.go

@@ -0,0 +1,8 @@
+package lfsapi
+
+import "crypto/x509"
+
+func appendRootCAsForHostFromPlatform(pool *x509.CertPool, host string) *x509.CertPool {
+	// Do nothing, use golang default
+	return pool
+}

lfsapi/certs_linux.go

@@ -0,0 +1,8 @@
+package lfsapi
+
+import "crypto/x509"
+
+func appendRootCAsForHostFromPlatform(pool *x509.CertPool, host string) *x509.CertPool {
+	// Do nothing, use golang default
+	return pool
+}

lfsapi/certs_openbsd.go

@@ -0,0 +1,8 @@
+package lfsapi
+
+import "crypto/x509"
+
+func appendRootCAsForHostFromPlatform(pool *x509.CertPool, host string) *x509.CertPool {
+	// Do nothing, use golang default
+	return pool
+}

lfsapi/certs_test.go

@@ -0,0 +1,230 @@
+package lfsapi
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var testCert = `-----BEGIN CERTIFICATE-----
+MIIDyjCCArKgAwIBAgIJAMi9TouXnW+ZMA0GCSqGSIb3DQEBBQUAMEwxCzAJBgNV
+BAYTAlVTMRMwEQYDVQQIEwpTb21lLVN0YXRlMRAwDgYDVQQKEwdnaXQtbGZzMRYw
+FAYDVQQDEw1naXQtbGZzLmxvY2FsMB4XDTE2MDMwOTEwNTk1NFoXDTI2MDMwNzEw
+NTk1NFowTDELMAkGA1UEBhMCVVMxEzARBgNVBAgTClNvbWUtU3RhdGUxEDAOBgNV
+BAoTB2dpdC1sZnMxFjAUBgNVBAMTDWdpdC1sZnMubG9jYWwwggEiMA0GCSqGSIb3
+DQEBAQUAA4IBDwAwggEKAoIBAQCXmsI2w44nOsP7n3kL1Lz04U5FMZRErBSXLOE+
+dpd4tMpgrjOncJPD9NapHabsVIOnuVvMDuBbWYwU9PwbN4tjQzch8DRxBju6fCp/
+Pm+QF6p2Ga+NuSHWoVfNFuF2776aF9gSLC0rFnBekD3HCz+h6I5HFgHBvRjeVyAs
+PRw471Y28Je609SoYugxaQNzRvahP0Qf43tE74/WN3FTGXy1+iU+uXpfp8KxnsuB
+gfj+Wi6mPt8Q2utcA1j82dJ0K8ZbHSbllzmI+N/UuRLsbTUEdeFWYdZ0AlZNd/Vc
+PlOSeoExwvOHIuUasT/cLIrEkdXNud2QLg2GpsB6fJi3NEUhAgMBAAGjga4wgasw
+HQYDVR0OBBYEFC8oVPRQbekTwfkntgdL7PADXNDbMHwGA1UdIwR1MHOAFC8oVPRQ
+bekTwfkntgdL7PADXNDboVCkTjBMMQswCQYDVQQGEwJVUzETMBEGA1UECBMKU29t
+ZS1TdGF0ZTEQMA4GA1UEChMHZ2l0LWxmczEWMBQGA1UEAxMNZ2l0LWxmcy5sb2Nh
+bIIJAMi9TouXnW+ZMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBACIl
+/CBLIhC3drrYme4cGArhWyXIyRpMoy9Z+9Dru8rSuOr/RXR6sbYhlE1iMGg4GsP8
+4Cj7aIct6Vb9NFv5bGNyFJAmDesm3SZlEcWxU3YBzNPiJXGiUpQHCkp0BH+gvsXc
+tb58XoiDZPVqrl0jNfX/nHpHR9c3DaI3Tjx0F/No0ZM6mLQ1cNMikFyEWQ4U0zmW
+LvV+vvKuOixRqbcVnB5iTxqMwFG0X3tUql0cftGBgoCoR1+FSBOs0EXLODCck6ql
+aW6vZwkA+ccj/pDTx8LBe2lnpatrFeIt6znAUJW3G8r6SFHKVBWHwmESZS4kxhjx
+NpW5Hh0w4/5iIetCkJ0=
+-----END CERTIFICATE-----`
+
+var sslCAInfoConfigHostNames = []string{
+	"git-lfs.local",
+	"git-lfs.local/",
+}
+var sslCAInfoMatchedHostTests = []struct {
+	hostName    string
+	shouldMatch bool
+}{
+	{"git-lfs.local", true},
+	{"git-lfs.local:8443", false},
+	{"wronghost.com", false},
+}
+
+func TestCertFromSSLCAInfoConfig(t *testing.T) {
+	tempfile, err := ioutil.TempFile("", "testcert")
+	assert.Nil(t, err, "Error creating temp cert file")
+	defer os.Remove(tempfile.Name())
+
+	_, err = tempfile.WriteString(testCert)
+	assert.Nil(t, err, "Error writing temp cert file")
+	tempfile.Close()
+
+	// Test http.<url>.sslcainfo
+	for _, hostName := range sslCAInfoConfigHostNames {
+		hostKey := fmt.Sprintf("http.https://%v.sslcainfo", hostName)
+		c, err := NewClient(nil, testEnv(map[string]string{
+			hostKey: tempfile.Name(),
+		}))
+		assert.Nil(t, err)
+
+		for _, matchedHostTest := range sslCAInfoMatchedHostTests {
+			pool := getRootCAsForHost(c, matchedHostTest.hostName)
+
+			var shouldOrShouldnt string
+			if matchedHostTest.shouldMatch {
+				shouldOrShouldnt = "should"
+			} else {
+				shouldOrShouldnt = "should not"
+			}
+
+			assert.Equal(t, matchedHostTest.shouldMatch, pool != nil,
+				"Cert lookup for \"%v\" %v have succeeded with \"%v\"",
+				matchedHostTest.hostName, shouldOrShouldnt, hostKey)
+		}
+	}
+
+	// Test http.sslcainfo
+	c, err := NewClient(nil, testEnv(map[string]string{
+		"http.sslcainfo": tempfile.Name(),
+	}))
+	assert.Nil(t, err)
+
+	// Should match any host at all
+	for _, matchedHostTest := range sslCAInfoMatchedHostTests {
+		pool := getRootCAsForHost(c, matchedHostTest.hostName)
+		assert.NotNil(t, pool)
+	}
+}
+
+func TestCertFromSSLCAInfoEnv(t *testing.T) {
+	tempfile, err := ioutil.TempFile("", "testcert")
+	assert.Nil(t, err, "Error creating temp cert file")
+	defer os.Remove(tempfile.Name())
+
+	_, err = tempfile.WriteString(testCert)
+	assert.Nil(t, err, "Error writing temp cert file")
+	tempfile.Close()
+
+	c, err := NewClient(testEnv(map[string]string{
+		"GIT_SSL_CAINFO": tempfile.Name(),
+	}), nil)
+	assert.Nil(t, err)
+
+	// Should match any host at all
+	for _, matchedHostTest := range sslCAInfoMatchedHostTests {
+		pool := getRootCAsForHost(c, matchedHostTest.hostName)
+		assert.NotNil(t, pool)
+	}
+}
+
+func TestCertFromSSLCAPathConfig(t *testing.T) {
+	tempdir, err := ioutil.TempDir("", "testcertdir")
+	assert.Nil(t, err, "Error creating temp cert dir")
+	defer os.RemoveAll(tempdir)
+
+	err = ioutil.WriteFile(filepath.Join(tempdir, "cert1.pem"), []byte(testCert), 0644)
+	assert.Nil(t, err, "Error creating cert file")
+
+	c, err := NewClient(nil, testEnv(map[string]string{
+		"http.sslcapath": tempdir,
+	}))
+
+	assert.Nil(t, err)
+
+	// Should match any host at all
+	for _, matchedHostTest := range sslCAInfoMatchedHostTests {
+		pool := getRootCAsForHost(c, matchedHostTest.hostName)
+		assert.NotNil(t, pool)
+	}
+}
+
+func TestCertFromSSLCAPathEnv(t *testing.T) {
+	tempdir, err := ioutil.TempDir("", "testcertdir")
+	assert.Nil(t, err, "Error creating temp cert dir")
+	defer os.RemoveAll(tempdir)
+
+	err = ioutil.WriteFile(filepath.Join(tempdir, "cert1.pem"), []byte(testCert), 0644)
+	assert.Nil(t, err, "Error creating cert file")
+
+	c, err := NewClient(testEnv(map[string]string{
+		"GIT_SSL_CAPATH": tempdir,
+	}), nil)
+	assert.Nil(t, err)
+
+	// Should match any host at all
+	for _, matchedHostTest := range sslCAInfoMatchedHostTests {
+		pool := getRootCAsForHost(c, matchedHostTest.hostName)
+		assert.NotNil(t, pool)
+	}
+}
+
+func TestCertVerifyDisabledGlobalEnv(t *testing.T) {
+	empty := &Client{}
+	httpClient := empty.httpClient("anyhost.com")
+	tr, ok := httpClient.Transport.(*http.Transport)
+	if assert.True(t, ok) {
+		assert.False(t, tr.TLSClientConfig.InsecureSkipVerify)
+	}
+
+	c, err := NewClient(testEnv(map[string]string{
+		"GIT_SSL_NO_VERIFY": "1",
+	}), nil)
+
+	assert.Nil(t, err)
+
+	httpClient = c.httpClient("anyhost.com")
+	tr, ok = httpClient.Transport.(*http.Transport)
+	if assert.True(t, ok) {
+		assert.True(t, tr.TLSClientConfig.InsecureSkipVerify)
+	}
+}
+
+func TestCertVerifyDisabledGlobalConfig(t *testing.T) {
+	def := &Client{}
+	httpClient := def.httpClient("anyhost.com")
+	tr, ok := httpClient.Transport.(*http.Transport)
+	if assert.True(t, ok) {
+		assert.False(t, tr.TLSClientConfig.InsecureSkipVerify)
+	}
+
+	c, err := NewClient(nil, testEnv(map[string]string{
+		"http.sslverify": "false",
+	}))
+	assert.Nil(t, err)
+
+	httpClient = c.httpClient("anyhost.com")
+	tr, ok = httpClient.Transport.(*http.Transport)
+	if assert.True(t, ok) {
+		assert.True(t, tr.TLSClientConfig.InsecureSkipVerify)
+	}
+}
+
+func TestCertVerifyDisabledHostConfig(t *testing.T) {
+	def := &Client{}
+	httpClient := def.httpClient("specifichost.com")
+	tr, ok := httpClient.Transport.(*http.Transport)
+	if assert.True(t, ok) {
+		assert.False(t, tr.TLSClientConfig.InsecureSkipVerify)
+	}
+
+	httpClient = def.httpClient("otherhost.com")
+	tr, ok = httpClient.Transport.(*http.Transport)
+	if assert.True(t, ok) {
+		assert.False(t, tr.TLSClientConfig.InsecureSkipVerify)
+	}
+
+	c, err := NewClient(nil, testEnv(map[string]string{
+		"http.https://specifichost.com/.sslverify": "false",
+	}))
+	assert.Nil(t, err)
+
+	httpClient = c.httpClient("specifichost.com")
+	tr, ok = httpClient.Transport.(*http.Transport)
+	if assert.True(t, ok) {
+		assert.True(t, tr.TLSClientConfig.InsecureSkipVerify)
+	}
+
+	httpClient = c.httpClient("otherhost.com")
+	tr, ok = httpClient.Transport.(*http.Transport)
+	if assert.True(t, ok) {
+		assert.False(t, tr.TLSClientConfig.InsecureSkipVerify)
+	}
+}

lfsapi/certs_windows.go

@@ -0,0 +1,8 @@
+package lfsapi
+
+import "crypto/x509"
+
+func appendRootCAsForHostFromPlatform(pool *x509.CertPool, host string) *x509.CertPool {
+	// golang already supports Windows Certificate Store for self-signed certs
+	return pool
+}

lfsapi/client_test.go

@@ -21,3 +21,51 @@ func TestNewClient(t *testing.T) {
 	assert.Equal(t, 153, c.TLSTimeout)
 	assert.Equal(t, 154, c.ConcurrentTransfers)
 }
+
+func TestNewClientWithGitSSLVerify(t *testing.T) {
+	c, err := NewClient(nil, nil)
+	assert.Nil(t, err)
+	assert.False(t, c.SkipSSLVerify)
+
+	for _, value := range []string{"true", "1", "t"} {
+		c, err = NewClient(testEnv(map[string]string{}), testEnv(map[string]string{
+			"http.sslverify": value,
+		}))
+		t.Logf("http.sslverify: %q", value)
+		assert.Nil(t, err)
+		assert.False(t, c.SkipSSLVerify)
+	}
+
+	for _, value := range []string{"false", "0", "f"} {
+		c, err = NewClient(testEnv(map[string]string{}), testEnv(map[string]string{
+			"http.sslverify": value,
+		}))
+		t.Logf("http.sslverify: %q", value)
+		assert.Nil(t, err)
+		assert.True(t, c.SkipSSLVerify)
+	}
+}
+
+func TestNewClientWithOSSSLVerify(t *testing.T) {
+	c, err := NewClient(nil, nil)
+	assert.Nil(t, err)
+	assert.False(t, c.SkipSSLVerify)
+
+	for _, value := range []string{"false", "0", "f"} {
+		c, err = NewClient(testEnv(map[string]string{
+			"GIT_SSL_NO_VERIFY": value,
+		}), testEnv(map[string]string{}))
+		t.Logf("GIT_SSL_NO_VERIFY: %q", value)
+		assert.Nil(t, err)
+		assert.False(t, c.SkipSSLVerify)
+	}
+
+	for _, value := range []string{"true", "1", "t"} {
+		c, err = NewClient(testEnv(map[string]string{
+			"GIT_SSL_NO_VERIFY": value,
+		}), testEnv(map[string]string{}))
+		t.Logf("GIT_SSL_NO_VERIFY: %q", value)
+		assert.Nil(t, err)
+		assert.True(t, c.SkipSSLVerify)
+	}
+}

lfsapi/lfsapi.go

@@ -1,6 +1,7 @@
 package lfsapi
 
 import (
+	"crypto/tls"
 	"encoding/json"
 	"net"
 	"net/http"
@@ -30,9 +31,14 @@ type Client struct {
 	HTTPSProxy          string
 	HTTPProxy           string
 	NoProxy             string
+	SkipSSLVerify       bool
 
 	hostClients map[string]*http.Client
 	clientMu    sync.Mutex
+
+	// only used for per-host ssl certs
+	gitEnv env
+	osEnv  env
 }
 
 func NewClient(osEnv env, gitEnv env) (*Client, error) {
@@ -61,9 +67,12 @@ func NewClient(osEnv env, gitEnv env) (*Client, error) {
 		KeepaliveTimeout:    gitEnv.Int("lfs.keepalive", 0),
 		TLSTimeout:          gitEnv.Int("lfs.tlstimeout", 0),
 		ConcurrentTransfers: gitEnv.Int("lfs.concurrenttransfers", 0),
+		SkipSSLVerify:       !gitEnv.Bool("http.sslverify", true) || osEnv.Bool("GIT_SSL_NO_VERIFY", false),
 		HTTPSProxy:          httpsProxy,
 		HTTPProxy:           httpProxy,
 		NoProxy:             noProxy,
+		gitEnv:              gitEnv,
+		osEnv:               osEnv,
 	}
 
 	return c, nil
@@ -82,6 +91,14 @@ func (c *Client) httpClient(host string) *http.Client {
 	c.clientMu.Lock()
 	defer c.clientMu.Unlock()
 
+	if c.gitEnv == nil {
+		c.gitEnv = make(testEnv)
+	}
+
+	if c.osEnv == nil {
+		c.osEnv = make(testEnv)
+	}
+
 	if c.hostClients == nil {
 		c.hostClients = make(map[string]*http.Client)
 	}
@@ -120,6 +137,13 @@ func (c *Client) httpClient(host string) *http.Client {
 		MaxIdleConnsPerHost: concurrentTransfers,
 	}
 
+	tr.TLSClientConfig = &tls.Config{}
+	if isCertVerificationDisabledForHost(c, host) {
+		tr.TLSClientConfig.InsecureSkipVerify = true
+	} else {
+		tr.TLSClientConfig.RootCAs = getRootCAsForHost(c, host)
+	}
+
 	httpClient := &http.Client{
 		Transport: tr,
 	}