diff --git a/api/http_lifecycle_test.go b/api/http_lifecycle_test.go
index 72013a99..b0b42a49 100644
--- a/api/http_lifecycle_test.go
+++ b/api/http_lifecycle_test.go
@@ -8,14 +8,16 @@ import (
 
 	"github.com/git-lfs/git-lfs/api"
 	"github.com/git-lfs/git-lfs/config"
+	"github.com/git-lfs/git-lfs/lfsapi"
 	"github.com/stretchr/testify/assert"
 )
 
 func NewTestConfig() *config.Configuration {
 	c := config.NewFrom(config.Values{})
-	c.SetManualEndpoint(config.Endpoint{Url: "https://example.com"})
+	c.SetManualEndpoint(lfsapi.Endpoint{Url: "https://example.com"})
 	return c
 }
+
 func TestHttpLifecycleMakesRequestsAgainstAbsolutePath(t *testing.T) {
 	SetupTestCredentialsFunc()
 	defer RestoreCredentialsFunc()
diff --git a/api/v1.go b/api/v1.go
index 24de5884..e754f821 100644
--- a/api/v1.go
+++ b/api/v1.go
@@ -9,6 +9,7 @@ import (
 	"github.com/git-lfs/git-lfs/config"
 	"github.com/git-lfs/git-lfs/errors"
 	"github.com/git-lfs/git-lfs/httputil"
+	"github.com/git-lfs/git-lfs/lfsapi"
 
 	"github.com/rubyist/tracerx"
 )
@@ -129,8 +130,8 @@ func NewBatchRequest(cfg *config.Configuration, operation string) (*http.Request
 	return req, nil
 }
 
-func ObjectUrl(endpoint config.Endpoint, oid string) (*url.URL, error) {
-	u, err := url.Parse(endpoint.Url)
+func ObjectUrl(e lfsapi.Endpoint, oid string) (*url.URL, error) {
+	u, err := url.Parse(e.Url)
 	if err != nil {
 		return nil, err
 	}
diff --git a/auth/ssh.go b/auth/ssh.go
index a09e4ceb..9ea86114 100644
--- a/auth/ssh.go
+++ b/auth/ssh.go
@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/git-lfs/git-lfs/config"
+	"github.com/git-lfs/git-lfs/lfsapi"
 	"github.com/rubyist/tracerx"
 )
 
@@ -19,7 +20,7 @@ type SshAuthResponse struct {
 	ExpiresAt string            `json:"expires_at"`
 }
 
-func SshAuthenticate(cfg *config.Configuration, operation, oid string) (SshAuthResponse, config.Endpoint, error) {
+func SshAuthenticate(cfg *config.Configuration, operation, oid string) (SshAuthResponse, lfsapi.Endpoint, error) {
 	// This is only used as a fallback where the Git URL is SSH but server doesn't support a full SSH binary protocol
 	// and therefore we derive a HTTPS endpoint for binaries instead; but check authentication here via SSH
 
@@ -61,8 +62,8 @@ func SshAuthenticate(cfg *config.Configuration, operation, oid string) (SshAuthR
 
 // Return the executable name for ssh on this machine and the base args
 // Base args includes port settings, user/host, everything pre the command to execute
-func sshGetExeAndArgs(cfg *config.Configuration, endpoint config.Endpoint) (exe string, baseargs []string) {
-	if len(endpoint.SshUserAndHost) == 0 {
+func sshGetExeAndArgs(cfg *config.Configuration, e lfsapi.Endpoint) (exe string, baseargs []string) {
+	if len(e.SshUserAndHost) == 0 {
 		return "", nil
 	}
 
@@ -99,15 +100,15 @@ func sshGetExeAndArgs(cfg *config.Configuration, endpoint config.Endpoint) (exe
 		args = append(args, "-batch")
 	}
 
-	if len(endpoint.SshPort) > 0 {
+	if len(e.SshPort) > 0 {
 		if isPlink || isTortoise {
 			args = append(args, "-P")
 		} else {
 			args = append(args, "-p")
 		}
-		args = append(args, endpoint.SshPort)
+		args = append(args, e.SshPort)
 	}
-	args = append(args, endpoint.SshUserAndHost)
+	args = append(args, e.SshUserAndHost)
 
 	return ssh, args
 }
diff --git a/config/config.go b/config/config.go
index 756d2fab..14952a5e 100644
--- a/config/config.go
+++ b/config/config.go
@@ -5,17 +5,14 @@ package config
 import (
 	"errors"
 	"fmt"
-	"os"
 	"reflect"
 	"strconv"
-	"strings"
 	"sync"
 
 	"github.com/ThomsonReutersEikon/go-ntlm/ntlm"
 	"github.com/bgentry/go-netrc/netrc"
-	"github.com/git-lfs/git-lfs/git"
+	"github.com/git-lfs/git-lfs/lfsapi"
 	"github.com/git-lfs/git-lfs/tools"
-	"github.com/rubyist/tracerx"
 )
 
 var (
@@ -68,23 +65,16 @@ type Configuration struct {
 	loading        sync.Mutex // guards initialization of gitConfig and remotes
 	remotes        []string
 	extensions     map[string]Extension
-	manualEndpoint *Endpoint
+	manualEndpoint *lfsapi.Endpoint
 	parsedNetrc    netrcfinder
-	urlAliasesMap  map[string]string
-	urlAliasMu     sync.Mutex
+	endpointFinder lfsapi.EndpointFinder
+	endpointMu     sync.Mutex
 }
 
 func New() *Configuration {
-	c := &Configuration{
-		Os:            EnvironmentOf(NewOsFetcher()),
-		CurrentRemote: defaultRemote,
-		envVars:       make(map[string]string),
-	}
-
+	c := &Configuration{Os: EnvironmentOf(NewOsFetcher())}
 	c.Git = &gitEnvironment{config: c}
-	c.IsTracingHttp = c.Os.Bool("GIT_CURL_VERBOSE", false)
-	c.IsDebuggingHttp = c.Os.Bool("LFS_DEBUG_HTTP", false)
-	c.IsLoggingStats = c.Os.Bool("GIT_LOG_STATS", false)
+	initConfig(c)
 	return c
 }
 
@@ -103,12 +93,20 @@ type Values struct {
 //
 // This method should only be used during testing.
 func NewFrom(v Values) *Configuration {
-	return &Configuration{
+	c := &Configuration{
 		Os:  EnvironmentOf(mapFetcher(v.Os)),
 		Git: EnvironmentOf(mapFetcher(v.Git)),
-
-		envVars: make(map[string]string, 0),
 	}
+	initConfig(c)
+	return c
+}
+
+func initConfig(c *Configuration) {
+	c.CurrentRemote = defaultRemote
+	c.envVars = make(map[string]string)
+	c.IsTracingHttp = c.Os.Bool("GIT_CURL_VERBOSE", false)
+	c.IsDebuggingHttp = c.Os.Bool("LFS_DEBUG_HTTP", false)
+	c.IsLoggingStats = c.Os.Bool("GIT_LOG_STATS", false)
 }
 
 // Unmarshal unmarshals the *Configuration in context into all of `v`'s fields,
@@ -202,51 +200,19 @@ func (c *Configuration) parseTag(tag reflect.StructTag) (key string, env Environ
 // GitRemoteUrl returns the git clone/push url for a given remote (blank if not found)
 // the forpush argument is to cater for separate remote.name.pushurl settings
 func (c *Configuration) GitRemoteUrl(remote string, forpush bool) string {
-	if forpush {
-		if u, ok := c.Git.Get("remote." + remote + ".pushurl"); ok {
-			return u
-		}
-	}
-
-	if u, ok := c.Git.Get("remote." + remote + ".url"); ok {
-		return u
-	}
-
-	if err := git.ValidateRemote(remote); err == nil {
-		return remote
-	}
-
-	return ""
-
+	return c.endpointConfig().GitRemoteURL(remote, forpush)
 }
 
 // Manually set an Endpoint to use instead of deriving from Git config
-func (c *Configuration) SetManualEndpoint(e Endpoint) {
+func (c *Configuration) SetManualEndpoint(e lfsapi.Endpoint) {
 	c.manualEndpoint = &e
 }
 
-func (c *Configuration) Endpoint(operation string) Endpoint {
+func (c *Configuration) Endpoint(operation string) lfsapi.Endpoint {
 	if c.manualEndpoint != nil {
 		return *c.manualEndpoint
 	}
-
-	if operation == "upload" {
-		if url, ok := c.Git.Get("lfs.pushurl"); ok {
-			return NewEndpointWithConfig(url, c)
-		}
-	}
-
-	if url, ok := c.Git.Get("lfs.url"); ok {
-		return NewEndpointWithConfig(url, c)
-	}
-
-	if len(c.CurrentRemote) > 0 && c.CurrentRemote != defaultRemote {
-		if endpoint := c.RemoteEndpoint(c.CurrentRemote, operation); len(endpoint.Url) > 0 {
-			return endpoint
-		}
-	}
-
-	return c.RemoteEndpoint(defaultRemote, operation)
+	return c.endpointConfig().Endpoint(operation, c.CurrentRemote)
 }
 
 func (c *Configuration) ConcurrentTransfers() int {
@@ -283,7 +249,7 @@ func (c *Configuration) BatchTransfer() bool {
 }
 
 func (c *Configuration) NtlmAccess(operation string) bool {
-	return c.Access(operation) == "ntlm"
+	return c.Access(operation) == lfsapi.NTLMAccess
 }
 
 // PrivateAccess will retrieve the access value and return true if
@@ -291,17 +257,17 @@ func (c *Configuration) NtlmAccess(operation string) bool {
 // access, the http requests for the batch api will fetch the credentials
 // before running, otherwise the request will run without credentials.
 func (c *Configuration) PrivateAccess(operation string) bool {
-	return c.Access(operation) != "none"
+	return c.Access(operation) != lfsapi.NoneAccess
 }
 
 // Access returns the access auth type.
-func (c *Configuration) Access(operation string) string {
+func (c *Configuration) Access(operation string) lfsapi.Access {
 	return c.EndpointAccess(c.Endpoint(operation))
 }
 
 // SetAccess will set the private access flag in .git/config.
 func (c *Configuration) SetAccess(operation string, authType string) {
-	c.SetEndpointAccess(c.Endpoint(operation), authType)
+	c.endpointConfig().SetAccess(c.Endpoint(operation).Url, lfsapi.Access(authType))
 }
 
 func (c *Configuration) FindNetrcHost(host string) (*netrc.Machine, error) {
@@ -323,34 +289,8 @@ func (c *Configuration) SetNetrc(n netrcfinder) {
 	c.parsedNetrc = n
 }
 
-func (c *Configuration) EndpointAccess(e Endpoint) string {
-	key := fmt.Sprintf("lfs.%s.access", e.Url)
-	if v, ok := c.Git.Get(key); ok && len(v) > 0 {
-		lower := strings.ToLower(v)
-		if lower == "private" {
-			return "basic"
-		}
-		return lower
-	}
-	return "none"
-}
-
-func (c *Configuration) SetEndpointAccess(e Endpoint, authType string) {
-	c.loadGitConfig()
-
-	tracerx.Printf("setting repository access to %s", authType)
-	key := fmt.Sprintf("lfs.%s.access", e.Url)
-
-	// Modify the config cache because it's checked again in this process
-	// without being reloaded.
-	switch authType {
-	case "", "none":
-		git.Config.UnsetLocalKey("", key)
-		c.Git.del(key)
-	default:
-		git.Config.SetLocal("", key, authType)
-		c.Git.set(key, authType)
-	}
+func (c *Configuration) EndpointAccess(e lfsapi.Endpoint) lfsapi.Access {
+	return c.endpointConfig().AccessFor(e.Url)
 }
 
 func (c *Configuration) FetchIncludePaths() []string {
@@ -363,27 +303,8 @@ func (c *Configuration) FetchExcludePaths() []string {
 	return tools.CleanPaths(patterns, ",")
 }
 
-func (c *Configuration) RemoteEndpoint(remote, operation string) Endpoint {
-	if len(remote) == 0 {
-		remote = defaultRemote
-	}
-
-	// Support separate push URL if specified and pushing
-	if operation == "upload" {
-		if url, ok := c.Git.Get("remote." + remote + ".lfspushurl"); ok {
-			return NewEndpointWithConfig(url, c)
-		}
-	}
-	if url, ok := c.Git.Get("remote." + remote + ".lfsurl"); ok {
-		return NewEndpointWithConfig(url, c)
-	}
-
-	// finally fall back on git remote url (also supports pushurl)
-	if url := c.GitRemoteUrl(remote, operation == "upload"); url != "" {
-		return NewEndpointFromCloneURLWithConfig(url, c)
-	}
-
-	return Endpoint{}
+func (c *Configuration) RemoteEndpoint(remote, operation string) lfsapi.Endpoint {
+	return c.endpointConfig().RemoteEndpoint(operation, remote)
 }
 
 func (c *Configuration) Remotes() []string {
@@ -392,18 +313,23 @@ func (c *Configuration) Remotes() []string {
 	return c.remotes
 }
 
-// GitProtocol returns the protocol for the LFS API when converting from a
-// git:// remote url.
 func (c *Configuration) GitProtocol() string {
-	if value, ok := c.Git.Get("lfs.gitprotocol"); ok {
-		return value
+	return c.endpointConfig().GitProtocol()
+}
+
+func (c *Configuration) endpointConfig() lfsapi.EndpointFinder {
+	c.endpointMu.Lock()
+	defer c.endpointMu.Unlock()
+
+	if c.endpointFinder == nil {
+		c.endpointFinder = lfsapi.NewEndpointFinder(c.Git)
 	}
-	return "https"
+
+	return c.endpointFinder
 }
 
 func (c *Configuration) Extensions() map[string]Extension {
 	c.loadGitConfig()
-
 	return c.extensions
 }
 
@@ -412,50 +338,6 @@ func (c *Configuration) SortedExtensions() ([]Extension, error) {
 	return SortExtensions(c.Extensions())
 }
 
-func (c *Configuration) urlAliases() map[string]string {
-	c.urlAliasMu.Lock()
-	defer c.urlAliasMu.Unlock()
-
-	if c.urlAliasesMap == nil {
-		c.urlAliasesMap = make(map[string]string)
-		prefix := "url."
-		suffix := ".insteadof"
-		for gitkey, gitval := range c.Git.All() {
-			if strings.HasPrefix(gitkey, prefix) && strings.HasSuffix(gitkey, suffix) {
-				if _, ok := c.urlAliasesMap[gitval]; ok {
-					fmt.Fprintf(os.Stderr, "WARNING: Multiple 'url.*.insteadof' keys with the same alias: %q\n", gitval)
-				}
-				c.urlAliasesMap[gitval] = gitkey[len(prefix) : len(gitkey)-len(suffix)]
-			}
-		}
-	}
-
-	return c.urlAliasesMap
-}
-
-// ReplaceUrlAlias returns a url with a prefix from a `url.*.insteadof` git
-// config setting. If multiple aliases match, use the longest one.
-// See https://git-scm.com/docs/git-config for Git's docs.
-func (c *Configuration) ReplaceUrlAlias(rawurl string) string {
-	var longestalias string
-	aliases := c.urlAliases()
-	for alias, _ := range aliases {
-		if !strings.HasPrefix(rawurl, alias) {
-			continue
-		}
-
-		if longestalias < alias {
-			longestalias = alias
-		}
-	}
-
-	if len(longestalias) > 0 {
-		return aliases[longestalias] + rawurl[len(longestalias):]
-	}
-
-	return rawurl
-}
-
 func (c *Configuration) FetchPruneConfig() FetchPruneConfig {
 	f := &FetchPruneConfig{
 		FetchRecentRefsDays:           7,
diff --git a/config/config_test.go b/config/config_test.go
index 29576be6..03a3db02 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -7,266 +7,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestEndpointDefaultsToOrigin(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{"remote.origin.lfsurl": "abc"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "abc", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-}
-
-func TestEndpointOverridesOrigin(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{
-			"lfs.url":              "abc",
-			"remote.origin.lfsurl": "def",
-		},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "abc", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-}
-
-func TestEndpointNoOverrideDefaultRemote(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{
-			"remote.origin.lfsurl": "abc",
-			"remote.other.lfsurl":  "def",
-		},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "abc", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-}
-
-func TestEndpointUseAlternateRemote(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{
-			"remote.origin.lfsurl": "abc",
-			"remote.other.lfsurl":  "def",
-		},
-	})
-
-	cfg.CurrentRemote = "other"
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "def", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-}
-
-func TestEndpointAddsLfsSuffix(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{"remote.origin.url": "https://example.com/foo/bar"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-}
-
-func TestBareEndpointAddsLfsSuffix(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{"remote.origin.url": "https://example.com/foo/bar.git"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-}
-
-func TestEndpointSeparateClonePushUrl(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{
-			"remote.origin.url":     "https://example.com/foo/bar.git",
-			"remote.origin.pushurl": "https://readwrite.com/foo/bar.git"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-
-	endpoint = cfg.Endpoint("upload")
-	assert.Equal(t, "https://readwrite.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-}
-
-func TestEndpointOverriddenSeparateClonePushLfsUrl(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{
-			"remote.origin.url":        "https://example.com/foo/bar.git",
-			"remote.origin.pushurl":    "https://readwrite.com/foo/bar.git",
-			"remote.origin.lfsurl":     "https://examplelfs.com/foo/bar",
-			"remote.origin.lfspushurl": "https://readwritelfs.com/foo/bar"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "https://examplelfs.com/foo/bar", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-
-	endpoint = cfg.Endpoint("upload")
-	assert.Equal(t, "https://readwritelfs.com/foo/bar", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-}
-
-func TestEndpointGlobalSeparateLfsPush(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{
-			"lfs.url":     "https://readonly.com/foo/bar",
-			"lfs.pushurl": "https://write.com/foo/bar",
-		},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "https://readonly.com/foo/bar", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-
-	endpoint = cfg.Endpoint("upload")
-	assert.Equal(t, "https://write.com/foo/bar", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-}
-
-func TestSSHEndpointOverridden(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{
-			"remote.origin.url":    "git@example.com:foo/bar",
-			"remote.origin.lfsurl": "lfs",
-		},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "lfs", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-	assert.Equal(t, "", endpoint.SshPort)
-}
-
-func TestSSHEndpointAddsLfsSuffix(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{"remote.origin.url": "ssh://git@example.com/foo/bar"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
-	assert.Equal(t, "foo/bar", endpoint.SshPath)
-	assert.Equal(t, "", endpoint.SshPort)
-}
-
-func TestSSHCustomPortEndpointAddsLfsSuffix(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{"remote.origin.url": "ssh://git@example.com:9000/foo/bar"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
-	assert.Equal(t, "foo/bar", endpoint.SshPath)
-	assert.Equal(t, "9000", endpoint.SshPort)
-}
-
-func TestBareSSHEndpointAddsLfsSuffix(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{"remote.origin.url": "git@example.com:foo/bar.git"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
-	assert.Equal(t, "foo/bar.git", endpoint.SshPath)
-	assert.Equal(t, "", endpoint.SshPort)
-}
-
-func TestSSHEndpointFromGlobalLfsUrl(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{"lfs.url": "git@example.com:foo/bar.git"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "https://example.com/foo/bar.git", endpoint.Url)
-	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
-	assert.Equal(t, "foo/bar.git", endpoint.SshPath)
-	assert.Equal(t, "", endpoint.SshPort)
-}
-
-func TestHTTPEndpointAddsLfsSuffix(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{"remote.origin.url": "http://example.com/foo/bar"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-	assert.Equal(t, "", endpoint.SshPort)
-}
-
-func TestBareHTTPEndpointAddsLfsSuffix(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{"remote.origin.url": "http://example.com/foo/bar.git"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-	assert.Equal(t, "", endpoint.SshPort)
-}
-
-func TestGitEndpointAddsLfsSuffix(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{"remote.origin.url": "git://example.com/foo/bar"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-	assert.Equal(t, "", endpoint.SshPort)
-}
-
-func TestGitEndpointAddsLfsSuffixWithCustomProtocol(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{
-			"remote.origin.url": "git://example.com/foo/bar",
-			"lfs.gitprotocol":   "http",
-		},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-	assert.Equal(t, "", endpoint.SshPort)
-}
-
-func TestBareGitEndpointAddsLfsSuffix(t *testing.T) {
-	cfg := NewFrom(Values{
-		Git: map[string]string{"remote.origin.url": "git://example.com/foo/bar.git"},
-	})
-
-	endpoint := cfg.Endpoint("download")
-	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
-	assert.Equal(t, "", endpoint.SshUserAndHost)
-	assert.Equal(t, "", endpoint.SshPath)
-	assert.Equal(t, "", endpoint.SshPort)
-}
-
 func TestConcurrentTransfersSetValue(t *testing.T) {
 	cfg := NewFrom(Values{
 		Git: map[string]string{
@@ -405,82 +145,6 @@ func TestBatchAbsentIsTrue(t *testing.T) {
 	assert.True(t, v)
 }
 
-func TestAccessConfig(t *testing.T) {
-	type accessTest struct {
-		Access        string
-		PrivateAccess bool
-	}
-
-	tests := map[string]accessTest{
-		"":            {"none", false},
-		"basic":       {"basic", true},
-		"BASIC":       {"basic", true},
-		"private":     {"basic", true},
-		"PRIVATE":     {"basic", true},
-		"invalidauth": {"invalidauth", true},
-	}
-
-	for value, expected := range tests {
-		cfg := NewFrom(Values{
-			Git: map[string]string{
-				"lfs.url":                        "http://example.com",
-				"lfs.http://example.com.access":  value,
-				"lfs.https://example.com.access": "bad",
-			},
-		})
-
-		if access := cfg.Access("download"); access != expected.Access {
-			t.Errorf("Expected Access() with value %q to be %v, got %v", value, expected.Access, access)
-		}
-		if access := cfg.Access("upload"); access != expected.Access {
-			t.Errorf("Expected Access() with value %q to be %v, got %v", value, expected.Access, access)
-		}
-
-		if priv := cfg.PrivateAccess("download"); priv != expected.PrivateAccess {
-			t.Errorf("Expected PrivateAccess() with value %q to be %v, got %v", value, expected.PrivateAccess, priv)
-		}
-		if priv := cfg.PrivateAccess("upload"); priv != expected.PrivateAccess {
-			t.Errorf("Expected PrivateAccess() with value %q to be %v, got %v", value, expected.PrivateAccess, priv)
-		}
-	}
-
-	// Test again but with separate push url
-	for value, expected := range tests {
-		cfg := NewFrom(Values{
-			Git: map[string]string{
-				"lfs.url":                           "http://example.com",
-				"lfs.pushurl":                       "http://examplepush.com",
-				"lfs.http://example.com.access":     value,
-				"lfs.http://examplepush.com.access": value,
-				"lfs.https://example.com.access":    "bad",
-			},
-		})
-
-		if access := cfg.Access("download"); access != expected.Access {
-			t.Errorf("Expected Access() with value %q to be %v, got %v", value, expected.Access, access)
-		}
-		if access := cfg.Access("upload"); access != expected.Access {
-			t.Errorf("Expected Access() with value %q to be %v, got %v", value, expected.Access, access)
-		}
-
-		if priv := cfg.PrivateAccess("download"); priv != expected.PrivateAccess {
-			t.Errorf("Expected PrivateAccess() with value %q to be %v, got %v", value, expected.PrivateAccess, priv)
-		}
-		if priv := cfg.PrivateAccess("upload"); priv != expected.PrivateAccess {
-			t.Errorf("Expected PrivateAccess() with value %q to be %v, got %v", value, expected.PrivateAccess, priv)
-		}
-	}
-
-}
-
-func TestAccessAbsentConfig(t *testing.T) {
-	cfg := NewFrom(Values{})
-	assert.Equal(t, "none", cfg.Access("download"))
-	assert.Equal(t, "none", cfg.Access("upload"))
-	assert.False(t, cfg.PrivateAccess("download"))
-	assert.False(t, cfg.PrivateAccess("upload"))
-}
-
 func TestLoadValidExtension(t *testing.T) {
 	cfg := NewFrom(Values{
 		Git: map[string]string{},
diff --git a/lfs/lfs.go b/lfs/lfs.go
index cbf48117..8c3c421a 100644
--- a/lfs/lfs.go
+++ b/lfs/lfs.go
@@ -120,7 +120,7 @@ func Environ(cfg *config.Configuration, manifest *tq.Manifest) []string {
 
 // TransferManifest builds a tq.Manifest using the given cfg.
 func TransferManifest(cfg *config.Configuration) *tq.Manifest {
-	return tq.NewManifestWithGitEnv(cfg.Access("download"), cfg.Git)
+	return tq.NewManifestWithGitEnv(string(cfg.Access("download")), cfg.Git)
 }
 
 func InRepo() bool {
diff --git a/config/endpoint.go b/lfsapi/endpoint.go
similarity index 56%
rename from config/endpoint.go
rename to lfsapi/endpoint.go
index d54a79e5..5464d7bb 100644
--- a/config/endpoint.go
+++ b/lfsapi/endpoint.go
@@ -1,14 +1,13 @@
-package config
+package lfsapi
 
 import (
 	"fmt"
 	"net/url"
-	"path"
 	"regexp"
 	"strings"
 )
 
-const EndpointUrlUnknown = "<unknown>"
+const UrlUnknown = "<unknown>"
 
 // An Endpoint describes how to access a Git LFS server.
 type Endpoint struct {
@@ -18,62 +17,6 @@ type Endpoint struct {
 	SshPort        string
 }
 
-// NewEndpointFromCloneURL creates an Endpoint from a git clone URL by appending
-// "[.git]/info/lfs".
-func NewEndpointFromCloneURL(url string) Endpoint {
-	return NewEndpointFromCloneURLWithConfig(url, New())
-}
-
-// NewEndpoint initializes a new Endpoint for a given URL.
-func NewEndpoint(rawurl string) Endpoint {
-	return NewEndpointWithConfig(rawurl, New())
-}
-
-// NewEndpointFromCloneURLWithConfig creates an Endpoint from a git clone URL by appending
-// "[.git]/info/lfs".
-func NewEndpointFromCloneURLWithConfig(url string, c *Configuration) Endpoint {
-	e := NewEndpointWithConfig(url, c)
-	if e.Url == EndpointUrlUnknown {
-		return e
-	}
-
-	if strings.HasSuffix(url, "/") {
-		e.Url = url[0 : len(url)-1]
-	}
-
-	// When using main remote URL for HTTP, append info/lfs
-	if path.Ext(e.Url) == ".git" {
-		e.Url += "/info/lfs"
-	} else {
-		e.Url += ".git/info/lfs"
-	}
-
-	return e
-}
-
-// NewEndpointWithConfig initializes a new Endpoint for a given URL.
-func NewEndpointWithConfig(rawurl string, c *Configuration) Endpoint {
-	rawurl = c.ReplaceUrlAlias(rawurl)
-	u, err := url.Parse(rawurl)
-	if err != nil {
-		return endpointFromBareSshUrl(rawurl)
-	}
-
-	switch u.Scheme {
-	case "ssh":
-		return endpointFromSshUrl(u)
-	case "http", "https":
-		return endpointFromHttpUrl(u)
-	case "git":
-		return endpointFromGitUrl(u, c)
-	case "":
-		return endpointFromBareSshUrl(u.String())
-	default:
-		// Just passthrough to preserve
-		return Endpoint{Url: rawurl}
-	}
-}
-
 // endpointFromBareSshUrl constructs a new endpoint from a bare SSH URL:
 //
 //   user@host.com:path/to/repo.git
@@ -95,7 +38,7 @@ func endpointFromBareSshUrl(rawurl string) Endpoint {
 	newrawurl := fmt.Sprintf("ssh://%v", newPath)
 	newu, err := url.Parse(newrawurl)
 	if err != nil {
-		return Endpoint{Url: EndpointUrlUnknown}
+		return Endpoint{Url: UrlUnknown}
 	}
 
 	return endpointFromSshUrl(newu)
@@ -108,7 +51,7 @@ func endpointFromSshUrl(u *url.URL) Endpoint {
 	regex := regexp.MustCompile(`^([^\:]+)(?:\:(\d+))?$`)
 	match := regex.FindStringSubmatch(u.Host)
 	if match == nil || len(match) < 2 {
-		endpoint.Url = EndpointUrlUnknown
+		endpoint.Url = UrlUnknown
 		return endpoint
 	}
 
@@ -145,7 +88,7 @@ func endpointFromHttpUrl(u *url.URL) Endpoint {
 	return Endpoint{Url: u.String()}
 }
 
-func endpointFromGitUrl(u *url.URL, c *Configuration) Endpoint {
-	u.Scheme = c.GitProtocol()
+func endpointFromGitUrl(u *url.URL, e *endpointGitFinder) Endpoint {
+	u.Scheme = e.gitProtocol
 	return Endpoint{Url: u.String()}
 }
diff --git a/lfsapi/endpoint_finder.go b/lfsapi/endpoint_finder.go
new file mode 100644
index 00000000..2ed0f37d
--- /dev/null
+++ b/lfsapi/endpoint_finder.go
@@ -0,0 +1,270 @@
+package lfsapi
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"sync"
+
+	"github.com/git-lfs/git-lfs/git"
+	"github.com/rubyist/tracerx"
+)
+
+type Access string
+
+const (
+	NoneAccess    Access = "none"
+	BasicAccess   Access = "basic"
+	PrivateAccess Access = "private"
+	NTLMAccess    Access = "ntlm"
+	emptyAccess   Access = ""
+	defaultRemote        = "origin"
+)
+
+type EndpointFinder interface {
+	NewEndpointFromCloneURL(rawurl string) Endpoint
+	NewEndpoint(rawurl string) Endpoint
+	Endpoint(operation, remote string) Endpoint
+	RemoteEndpoint(operation, remote string) Endpoint
+	GitRemoteURL(remote string, forpush bool) string
+	AccessFor(rawurl string) Access
+	SetAccess(rawurl string, access Access)
+	GitProtocol() string
+}
+
+type endpointGitFinder struct {
+	git         env
+	gitProtocol string
+
+	aliasMu sync.Mutex
+	aliases map[string]string
+
+	accessMu  sync.Mutex
+	urlAccess map[string]Access
+}
+
+func NewEndpointFinder(git env) EndpointFinder {
+	e := &endpointGitFinder{
+		gitProtocol: "https",
+		aliases:     make(map[string]string),
+		urlAccess:   make(map[string]Access),
+	}
+
+	if git != nil {
+		e.git = git
+		if v, ok := git.Get("lfs.gitprotocol"); ok {
+			e.gitProtocol = v
+		}
+		initAliases(e, git)
+	}
+
+	return e
+}
+
+func (e *endpointGitFinder) Endpoint(operation, remote string) Endpoint {
+	if e.git == nil {
+		return Endpoint{}
+	}
+
+	if operation == "upload" {
+		if url, ok := e.git.Get("lfs.pushurl"); ok {
+			return e.NewEndpoint(url)
+		}
+	}
+
+	if url, ok := e.git.Get("lfs.url"); ok {
+		return e.NewEndpoint(url)
+	}
+
+	if len(remote) > 0 && remote != defaultRemote {
+		if e := e.RemoteEndpoint(operation, remote); len(e.Url) > 0 {
+			return e
+		}
+	}
+
+	return e.RemoteEndpoint(operation, defaultRemote)
+}
+
+func (e *endpointGitFinder) RemoteEndpoint(operation, remote string) Endpoint {
+	if e.git == nil {
+		return Endpoint{}
+	}
+
+	if len(remote) == 0 {
+		remote = defaultRemote
+	}
+
+	// Support separate push URL if specified and pushing
+	if operation == "upload" {
+		if url, ok := e.git.Get("remote." + remote + ".lfspushurl"); ok {
+			return e.NewEndpoint(url)
+		}
+	}
+	if url, ok := e.git.Get("remote." + remote + ".lfsurl"); ok {
+		return e.NewEndpoint(url)
+	}
+
+	// finally fall back on git remote url (also supports pushurl)
+	if url := e.GitRemoteURL(remote, operation == "upload"); url != "" {
+		return e.NewEndpointFromCloneURL(url)
+	}
+
+	return Endpoint{}
+}
+
+func (e *endpointGitFinder) GitRemoteURL(remote string, forpush bool) string {
+	if e.git != nil {
+		if forpush {
+			if u, ok := e.git.Get("remote." + remote + ".pushurl"); ok {
+				return u
+			}
+		}
+
+		if u, ok := e.git.Get("remote." + remote + ".url"); ok {
+			return u
+		}
+	}
+
+	if err := git.ValidateRemote(remote); err == nil {
+		return remote
+	}
+
+	return ""
+}
+
+func (e *endpointGitFinder) NewEndpointFromCloneURL(rawurl string) Endpoint {
+	ep := e.NewEndpoint(rawurl)
+	if ep.Url == UrlUnknown {
+		return ep
+	}
+
+	if strings.HasSuffix(rawurl, "/") {
+		ep.Url = rawurl[0 : len(rawurl)-1]
+	}
+
+	// When using main remote URL for HTTP, append info/lfs
+	if path.Ext(ep.Url) == ".git" {
+		ep.Url += "/info/lfs"
+	} else {
+		ep.Url += ".git/info/lfs"
+	}
+
+	return ep
+}
+
+func (e *endpointGitFinder) NewEndpoint(rawurl string) Endpoint {
+	rawurl = e.ReplaceUrlAlias(rawurl)
+	u, err := url.Parse(rawurl)
+	if err != nil {
+		return endpointFromBareSshUrl(rawurl)
+	}
+
+	switch u.Scheme {
+	case "ssh":
+		return endpointFromSshUrl(u)
+	case "http", "https":
+		return endpointFromHttpUrl(u)
+	case "git":
+		return endpointFromGitUrl(u, e)
+	case "":
+		return endpointFromBareSshUrl(u.String())
+	default:
+		// Just passthrough to preserve
+		return Endpoint{Url: rawurl}
+	}
+}
+
+func (e *endpointGitFinder) AccessFor(rawurl string) Access {
+	if e.git == nil {
+		return NoneAccess
+	}
+
+	e.accessMu.Lock()
+	defer e.accessMu.Unlock()
+
+	if cached, ok := e.urlAccess[rawurl]; ok {
+		return cached
+	}
+
+	key := fmt.Sprintf("lfs.%s.access", rawurl)
+	e.urlAccess[rawurl] = fetchGitAccess(e.git, key)
+	return e.urlAccess[rawurl]
+}
+
+func (e *endpointGitFinder) SetAccess(rawurl string, access Access) {
+	key := fmt.Sprintf("lfs.%s.access", rawurl)
+	tracerx.Printf("setting repository access to %s", access)
+
+	e.accessMu.Lock()
+	defer e.accessMu.Unlock()
+
+	switch access {
+	case emptyAccess, NoneAccess:
+		git.Config.UnsetLocalKey("", key)
+		e.urlAccess[rawurl] = NoneAccess
+	default:
+		git.Config.SetLocal("", key, string(access))
+		e.urlAccess[rawurl] = access
+	}
+}
+
+func fetchGitAccess(git env, key string) Access {
+	if v, _ := git.Get(key); len(v) > 0 {
+		access := Access(strings.ToLower(v))
+		if access == PrivateAccess {
+			return BasicAccess
+		}
+		return access
+	}
+	return NoneAccess
+}
+
+func (e *endpointGitFinder) GitProtocol() string {
+	return e.gitProtocol
+}
+
+// ReplaceUrlAlias returns a url with a prefix from a `url.*.insteadof` git
+// config setting. If multiple aliases match, use the longest one.
+// See https://git-scm.com/docs/git-config for Git's docs.
+func (e *endpointGitFinder) ReplaceUrlAlias(rawurl string) string {
+	e.aliasMu.Lock()
+	defer e.aliasMu.Unlock()
+
+	var longestalias string
+	for alias, _ := range e.aliases {
+		if !strings.HasPrefix(rawurl, alias) {
+			continue
+		}
+
+		if longestalias < alias {
+			longestalias = alias
+		}
+	}
+
+	if len(longestalias) > 0 {
+		return e.aliases[longestalias] + rawurl[len(longestalias):]
+	}
+
+	return rawurl
+}
+
+func initAliases(e *endpointGitFinder, git env) {
+	prefix := "url."
+	suffix := ".insteadof"
+	for gitkey, gitval := range git.All() {
+		if !(strings.HasPrefix(gitkey, prefix) && strings.HasSuffix(gitkey, suffix)) {
+			continue
+		}
+		if _, ok := e.aliases[gitval]; ok {
+			fmt.Fprintf(os.Stderr, "WARNING: Multiple 'url.*.insteadof' keys with the same alias: %q\n", gitval)
+		}
+		e.aliases[gitval] = gitkey[len(prefix) : len(gitkey)-len(suffix)]
+	}
+}
+
+type env interface {
+	Get(string) (string, bool)
+	All() map[string]string
+}
diff --git a/lfsapi/endpoint_finder_test.go b/lfsapi/endpoint_finder_test.go
new file mode 100644
index 00000000..bbe095e6
--- /dev/null
+++ b/lfsapi/endpoint_finder_test.go
@@ -0,0 +1,361 @@
+package lfsapi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEndpointDefaultsToOrigin(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.lfsurl": "abc",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "abc", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+}
+
+func TestEndpointOverridesOrigin(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"lfs.url":              "abc",
+		"remote.origin.lfsurl": "def",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "abc", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+}
+
+func TestEndpointNoOverrideDefaultRemote(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.lfsurl": "abc",
+		"remote.other.lfsurl":  "def",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "abc", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+}
+
+func TestEndpointUseAlternateRemote(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.lfsurl": "abc",
+		"remote.other.lfsurl":  "def",
+	}))
+
+	e := finder.Endpoint("download", "other")
+	assert.Equal(t, "def", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+}
+
+func TestEndpointAddsLfsSuffix(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url": "https://example.com/foo/bar",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+}
+
+func TestBareEndpointAddsLfsSuffix(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url": "https://example.com/foo/bar.git",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+}
+
+func TestEndpointSeparateClonePushUrl(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url":     "https://example.com/foo/bar.git",
+		"remote.origin.pushurl": "https://readwrite.com/foo/bar.git",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+
+	e = finder.Endpoint("upload", "")
+	assert.Equal(t, "https://readwrite.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+}
+
+func TestEndpointOverriddenSeparateClonePushLfsUrl(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url":        "https://example.com/foo/bar.git",
+		"remote.origin.pushurl":    "https://readwrite.com/foo/bar.git",
+		"remote.origin.lfsurl":     "https://examplelfs.com/foo/bar",
+		"remote.origin.lfspushurl": "https://readwritelfs.com/foo/bar",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "https://examplelfs.com/foo/bar", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+
+	e = finder.Endpoint("upload", "")
+	assert.Equal(t, "https://readwritelfs.com/foo/bar", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+}
+
+func TestEndpointGlobalSeparateLfsPush(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"lfs.url":     "https://readonly.com/foo/bar",
+		"lfs.pushurl": "https://write.com/foo/bar",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "https://readonly.com/foo/bar", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+
+	e = finder.Endpoint("upload", "")
+	assert.Equal(t, "https://write.com/foo/bar", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+}
+
+func TestSSHEndpointOverridden(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url":    "git@example.com:foo/bar",
+		"remote.origin.lfsurl": "lfs",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "lfs", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+	assert.Equal(t, "", e.SshPort)
+}
+
+func TestSSHEndpointAddsLfsSuffix(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url": "ssh://git@example.com/foo/bar",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "git@example.com", e.SshUserAndHost)
+	assert.Equal(t, "foo/bar", e.SshPath)
+	assert.Equal(t, "", e.SshPort)
+}
+
+func TestSSHCustomPortEndpointAddsLfsSuffix(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url": "ssh://git@example.com:9000/foo/bar",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "git@example.com", e.SshUserAndHost)
+	assert.Equal(t, "foo/bar", e.SshPath)
+	assert.Equal(t, "9000", e.SshPort)
+}
+
+func TestBareSSHEndpointAddsLfsSuffix(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url": "git@example.com:foo/bar.git",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "git@example.com", e.SshUserAndHost)
+	assert.Equal(t, "foo/bar.git", e.SshPath)
+	assert.Equal(t, "", e.SshPort)
+}
+
+func TestSSHEndpointFromGlobalLfsUrl(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"lfs.url": "git@example.com:foo/bar.git",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "https://example.com/foo/bar.git", e.Url)
+	assert.Equal(t, "git@example.com", e.SshUserAndHost)
+	assert.Equal(t, "foo/bar.git", e.SshPath)
+	assert.Equal(t, "", e.SshPort)
+}
+
+func TestHTTPEndpointAddsLfsSuffix(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url": "http://example.com/foo/bar",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+	assert.Equal(t, "", e.SshPort)
+}
+
+func TestBareHTTPEndpointAddsLfsSuffix(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url": "http://example.com/foo/bar.git",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+	assert.Equal(t, "", e.SshPort)
+}
+
+func TestGitEndpointAddsLfsSuffix(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url": "git://example.com/foo/bar",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+	assert.Equal(t, "", e.SshPort)
+}
+
+func TestGitEndpointAddsLfsSuffixWithCustomProtocol(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url": "git://example.com/foo/bar",
+		"lfs.gitprotocol":   "http",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+	assert.Equal(t, "", e.SshPort)
+}
+
+func TestBareGitEndpointAddsLfsSuffix(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"remote.origin.url": "git://example.com/foo/bar.git",
+	}))
+
+	e := finder.Endpoint("download", "")
+	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", e.Url)
+	assert.Equal(t, "", e.SshUserAndHost)
+	assert.Equal(t, "", e.SshPath)
+	assert.Equal(t, "", e.SshPort)
+}
+
+func TestAccessConfig(t *testing.T) {
+	type accessTest struct {
+		Access        string
+		PrivateAccess bool
+	}
+
+	tests := map[string]accessTest{
+		"":            {"none", false},
+		"basic":       {"basic", true},
+		"BASIC":       {"basic", true},
+		"private":     {"basic", true},
+		"PRIVATE":     {"basic", true},
+		"invalidauth": {"invalidauth", true},
+	}
+
+	for value, expected := range tests {
+		finder := NewEndpointFinder(gitEnv(map[string]string{
+			"lfs.url":                        "http://example.com",
+			"lfs.http://example.com.access":  value,
+			"lfs.https://example.com.access": "bad",
+		}))
+
+		dl := finder.Endpoint("upload", "")
+		ul := finder.Endpoint("download", "")
+
+		if access := finder.AccessFor(dl.Url); access != Access(expected.Access) {
+			t.Errorf("Expected Access() with value %q to be %v, got %v", value, expected.Access, access)
+		}
+		if access := finder.AccessFor(ul.Url); access != Access(expected.Access) {
+			t.Errorf("Expected Access() with value %q to be %v, got %v", value, expected.Access, access)
+		}
+	}
+
+	// Test again but with separate push url
+	for value, expected := range tests {
+		finder := NewEndpointFinder(gitEnv(map[string]string{
+			"lfs.url":                           "http://example.com",
+			"lfs.pushurl":                       "http://examplepush.com",
+			"lfs.http://example.com.access":     value,
+			"lfs.http://examplepush.com.access": value,
+			"lfs.https://example.com.access":    "bad",
+		}))
+
+		dl := finder.Endpoint("upload", "")
+		ul := finder.Endpoint("download", "")
+
+		if access := finder.AccessFor(dl.Url); access != Access(expected.Access) {
+			t.Errorf("Expected Access() with value %q to be %v, got %v", value, expected.Access, access)
+		}
+		if access := finder.AccessFor(ul.Url); access != Access(expected.Access) {
+			t.Errorf("Expected Access() with value %q to be %v, got %v", value, expected.Access, access)
+		}
+	}
+}
+
+func TestAccessAbsentConfig(t *testing.T) {
+	finder := NewEndpointFinder(nil)
+	assert.Equal(t, NoneAccess, finder.AccessFor(finder.Endpoint("download", "").Url))
+	assert.Equal(t, NoneAccess, finder.AccessFor(finder.Endpoint("upload", "").Url))
+}
+
+func TestSetAccess(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{}))
+
+	assert.Equal(t, NoneAccess, finder.AccessFor("http://example.com"))
+	finder.SetAccess("http://example.com", NTLMAccess)
+	assert.Equal(t, NTLMAccess, finder.AccessFor("http://example.com"))
+}
+
+func TestChangeAccess(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"lfs.http://example.com.access": "basic",
+	}))
+
+	assert.Equal(t, BasicAccess, finder.AccessFor("http://example.com"))
+	finder.SetAccess("http://example.com", NTLMAccess)
+	assert.Equal(t, NTLMAccess, finder.AccessFor("http://example.com"))
+}
+
+func TestDeleteAccessWithNone(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"lfs.http://example.com.access": "basic",
+	}))
+
+	assert.Equal(t, BasicAccess, finder.AccessFor("http://example.com"))
+	finder.SetAccess("http://example.com", NoneAccess)
+	assert.Equal(t, NoneAccess, finder.AccessFor("http://example.com"))
+}
+
+func TestDeleteAccessWithEmptyString(t *testing.T) {
+	finder := NewEndpointFinder(gitEnv(map[string]string{
+		"lfs.http://example.com.access": "basic",
+	}))
+
+	assert.Equal(t, BasicAccess, finder.AccessFor("http://example.com"))
+	finder.SetAccess("http://example.com", Access(""))
+	assert.Equal(t, NoneAccess, finder.AccessFor("http://example.com"))
+}
+
+type gitEnv map[string]string
+
+func (e gitEnv) Get(key string) (string, bool) {
+	v, ok := e[key]
+	return v, ok
+}
+
+func (e gitEnv) All() map[string]string {
+	return e
+}
diff --git a/config/endpoint_test.go b/lfsapi/endpoint_test.go
similarity index 79%
rename from config/endpoint_test.go
rename to lfsapi/endpoint_test.go
index af4c245d..5bf208bc 100644
--- a/config/endpoint_test.go
+++ b/lfsapi/endpoint_test.go
@@ -1,4 +1,4 @@
-package config
+package lfsapi
 
 import (
 	"testing"
@@ -13,9 +13,9 @@ func TestNewEndpointFromCloneURLWithConfig(t *testing.T) {
 		"https://foo/bar.git/",
 	}
 
-	cfg := New()
+	finder := NewEndpointFinder(nil)
 	for _, actual := range tests {
-		e := NewEndpointFromCloneURLWithConfig(actual, cfg)
+		e := finder.NewEndpointFromCloneURL(actual)
 		if e.Url != expected {
 			t.Errorf("%s returned bad endpoint url %s", actual, e.Url)
 		}
diff --git a/lfsapi/lfsapi.go b/lfsapi/lfsapi.go
index 4260f9bc..1e200de3 100644
--- a/lfsapi/lfsapi.go
+++ b/lfsapi/lfsapi.go
@@ -14,6 +14,7 @@ var (
 )
 
 type Client struct {
+	Endpoints EndpointFinder
 }
 
 func (c *Client) Do(req *http.Request) (*http.Response, error) {
diff --git a/test/git-lfs-test-server-api/main.go b/test/git-lfs-test-server-api/main.go
index c211cce7..f6cf26de 100644
--- a/test/git-lfs-test-server-api/main.go
+++ b/test/git-lfs-test-server-api/main.go
@@ -14,6 +14,7 @@ import (
 	"github.com/git-lfs/git-lfs/config"
 	"github.com/git-lfs/git-lfs/errors"
 	"github.com/git-lfs/git-lfs/lfs"
+	"github.com/git-lfs/git-lfs/lfsapi"
 	"github.com/git-lfs/git-lfs/progress"
 	"github.com/git-lfs/git-lfs/test"
 	"github.com/git-lfs/git-lfs/tq"
@@ -66,11 +67,12 @@ func testServerApi(cmd *cobra.Command, args []string) {
 	config.Config.Git.All()
 
 	// Configure the endpoint manually
-	var endp config.Endpoint
+	var endp lfsapi.Endpoint
+	finder := lfsapi.NewEndpointFinder(config.Config.Git)
 	if len(cloneUrl) > 0 {
-		endp = config.NewEndpointFromCloneURL(cloneUrl)
+		endp = finder.NewEndpointFromCloneURL(cloneUrl)
 	} else {
-		endp = config.NewEndpoint(apiUrl)
+		endp = finder.NewEndpoint(apiUrl)
 	}
 	config.Config.SetManualEndpoint(endp)