lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored many api functions to httputil/api package from lfs

Browse Source 
Also moved ntlm to httputil since too http-specific to be contained in auth
This commit is contained in:
Steve Streeting 2016-05-16 17:57:37 +01:00
parent 633044ad2d
commit 3298d9bbb3
11 changed files with 419 additions and 391 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
api/client.go
View File

@ -1 +1,62 @@
package api
import (
"net/http"
"github.com/github/git-lfs/auth"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/httputil"
)
// doLegacyApiRequest runs the request to the LFS legacy API.
func DoLegacyRequest(req *http.Request) (*http.Response, *ObjectResource, error) {
via := make([]*http.Request, 0, 4)
res, err := httputil.DoHttpRequestWithRedirects(req, via, true)
if err != nil {
return res, nil, err
}
obj := &ObjectResource{}
err = httputil.DecodeResponse(res, obj)
if err != nil {
httputil.SetErrorResponseContext(err, res)
return nil, nil, err
}
return res, obj, nil
}
// doApiBatchRequest runs the request to the LFS batch API. If the API returns a
// 401, the repo will be marked as having private access and the request will be
// re-run. When the repo is marked as having private access, credentials will
// be retrieved.
func DoBatchRequest(req *http.Request) (*http.Response, []*ObjectResource, error) {
res, err := DoRequest(req, config.Config.PrivateAccess(auth.GetOperationForRequest(req)))
if err != nil {
if res != nil && res.StatusCode == 401 {
return res, nil, errutil.NewAuthError(err)
}
return res, nil, err
}
var objs map[string][]*ObjectResource
err = httputil.DecodeResponse(res, &objs)
if err != nil {
httputil.SetErrorResponseContext(err, res)
}
return res, objs["objects"], err
}
// DoRequest runs a request to the LFS API, without parsing the response
// body. If the API returns a 401, the repo will be marked as having private
// access and the request will be re-run. When the repo is marked as having
// private access, credentials will be retrieved.
func DoRequest(req *http.Request, useCreds bool) (*http.Response, error) {
via := make([]*http.Request, 0, 4)
return httputil.DoHttpRequestWithRedirects(req, via, useCreds)
}

18
auth/credentials.go
View File

@ -14,7 +14,6 @@ import (
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/httputil"
"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)
@ -51,7 +50,7 @@ func GetCreds(req *http.Request) (Creds, error) {
}
func getCredURLForAPI(req *http.Request) (*url.URL, error) {
operation := httputil.GetOperationForRequest(req)
operation := GetOperationForRequest(req)
apiUrl, err := url.Parse(config.Config.Endpoint(operation).Url)
if err != nil {
return nil, err
@ -120,7 +119,7 @@ func setCredURLFromNetrc(req *http.Request) bool {
}
func skipCredsCheck(req *http.Request) bool {
if config.Config.NtlmAccess(httputil.GetOperationForRequest(req)) {
if config.Config.NtlmAccess(GetOperationForRequest(req)) {
return false
}
@ -242,7 +241,7 @@ func execCredsCommand(input Creds, subCommand string) (Creds, error) {
}
func setRequestAuthFromUrl(req *http.Request, u *url.URL) bool {
if !config.Config.NtlmAccess(httputil.GetOperationForRequest(req)) && u.User != nil {
if !config.Config.NtlmAccess(GetOperationForRequest(req)) && u.User != nil {
if pass, ok := u.User.Password(); ok {
fmt.Fprintln(os.Stderr, "warning: current Git remote contains credentials")
setRequestAuth(req, u.User.Username(), pass)
@ -254,7 +253,7 @@ func setRequestAuthFromUrl(req *http.Request, u *url.URL) bool {
}
func setRequestAuth(req *http.Request, user, pass string) {
if config.Config.NtlmAccess(httputil.GetOperationForRequest(req)) {
if config.Config.NtlmAccess(GetOperationForRequest(req)) {
return
}
@ -281,3 +280,12 @@ func SetCredentialsFunc(f CredentialFunc) CredentialFunc {
execCreds = f
return oldf
}
// GetOperationForRequest determines the operation type for a http.Request
func GetOperationForRequest(req *http.Request) string {
operation := "download"
if req.Method == "POST" || req.Method == "PUT" {
operation = "upload"
}
return operation
}

24
httputil/http.go
View File

@ -102,21 +102,6 @@ func (c *HttpClient) Do(req *http.Request) (*http.Response, error) {
return res, err
}
func NewHttpRequest(method, rawurl string, header map[string]string) (*http.Request, error) {
req, err := http.NewRequest(method, rawurl, nil)
if err != nil {
return nil, err
}
for key, value := range header {
req.Header.Set(key, value)
}
req.Header.Set("User-Agent", UserAgent)
return req, nil
}
// NewHttpClient returns a new HttpClient for the given host (which may be "host:port")
func NewHttpClient(c *config.Configuration, host string) *HttpClient {
httpClientsMutex.Lock()
@ -354,15 +339,6 @@ func TraceHttpReq(req *http.Request) string {
return fmt.Sprintf("%s %s", req.Method, strings.SplitN(req.URL.String(), "?", 2)[0])
}
// GetOperationForRequest determines the operation type for a http.Request
func GetOperationForRequest(req *http.Request) string {
operation := "download"
if req.Method == "POST" || req.Method == "PUT" {
operation = "upload"
}
return operation
}
func init() {
UserAgent = config.VersionDesc
}

22
auth/ntlm.go → httputil/ntlm.go
View File

@ -1,4 +1,4 @@
package auth
package httputil
import (
"bytes"
@ -12,12 +12,12 @@ import (
"strings"
"sync/atomic"
"github.com/github/git-lfs/auth"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/httputil"
"github.com/github/git-lfs/vendor/_nuts/github.com/ThomsonReutersEikon/go-ntlm/ntlm"
)
func ntlmClientSession(c *config.Configuration, creds Creds) (ntlm.ClientSession, error) {
func ntlmClientSession(c *config.Configuration, creds auth.Creds) (ntlm.ClientSession, error) {
if c.NtlmSession != nil {
return c.NtlmSession, nil
}
@ -39,13 +39,13 @@ func ntlmClientSession(c *config.Configuration, creds Creds) (ntlm.ClientSession
return session, nil
}
func DoNTLMRequest(request *http.Request, retry bool) (*http.Response, error) {
func doNTLMRequest(request *http.Request, retry bool) (*http.Response, error) {
handReq, err := cloneRequest(request)
if err != nil {
return nil, err
}
res, err := httputil.NewHttpClient(config.Config, handReq.Host).Do(handReq)
res, err := NewHttpClient(config.Config, handReq.Host).Do(handReq)
if err != nil && res == nil {
return nil, err
}
@ -53,7 +53,7 @@ func DoNTLMRequest(request *http.Request, retry bool) (*http.Response, error) {
//If the status is 401 then we need to re-authenticate, otherwise it was successful
if res.StatusCode == 401 {
creds, err := GetCreds(request)
creds, err := auth.GetCreds(request)
if err != nil {
return nil, err
}
@ -80,10 +80,10 @@ func DoNTLMRequest(request *http.Request, retry bool) (*http.Response, error) {
//If the status is 401 then we need to re-authenticate
if res.StatusCode == 401 && retry == true {
return DoNTLMRequest(challengeReq, false)
return doNTLMRequest(challengeReq, false)
}
SaveCredentials(creds, res)
auth.SaveCredentials(creds, res)
return res, nil
}
@ -92,7 +92,7 @@ func DoNTLMRequest(request *http.Request, retry bool) (*http.Response, error) {
func negotiate(request *http.Request, message string) ([]byte, error) {
request.Header.Add("Authorization", message)
res, err := httputil.NewHttpClient(config.Config, request.Host).Do(request)
res, err := NewHttpClient(config.Config, request.Host).Do(request)
if res == nil && err != nil {
return nil, err
@ -109,7 +109,7 @@ func negotiate(request *http.Request, message string) ([]byte, error) {
return ret, nil
}
func challenge(request *http.Request, challengeBytes []byte, creds Creds) (*http.Response, error) {
func challenge(request *http.Request, challengeBytes []byte, creds auth.Creds) (*http.Response, error) {
challenge, err := ntlm.ParseChallengeMessage(challengeBytes)
if err != nil {
return nil, err
@ -128,7 +128,7 @@ func challenge(request *http.Request, challengeBytes []byte, creds Creds) (*http
authMsg := base64.StdEncoding.EncodeToString(authenticate.Bytes())
request.Header.Add("Authorization", "NTLM "+authMsg)
return httputil.NewHttpClient(config.Config, request.Host).Do(request)
return NewHttpClient(config.Config, request.Host).Do(request)
}
func parseChallengeResponse(response *http.Response) ([]byte, error) {

2
auth/ntlm_test.go → httputil/ntlm_test.go
View File

@ -1,4 +1,4 @@
package auth
package httputil
import (
"bytes"

186
httputil/request.go Normal file
View File

@ -0,0 +1,186 @@
package httputil
import (
"bytes"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
"github.com/github/git-lfs/auth"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)
type ClientError struct {
Message string `json:"message"`
DocumentationUrl string `json:"documentation_url,omitempty"`
RequestId string `json:"request_id,omitempty"`
}
func (e *ClientError) Error() string {
msg := e.Message
if len(e.DocumentationUrl) > 0 {
msg += "\nDocs: " + e.DocumentationUrl
}
if len(e.RequestId) > 0 {
msg += "\nRequest ID: " + e.RequestId
}
return msg
}
// Internal http request management
func doHttpRequest(req *http.Request, creds auth.Creds) (*http.Response, error) {
var (
res *http.Response
err error
)
if config.Config.NtlmAccess(auth.GetOperationForRequest(req)) {
res, err = doNTLMRequest(req, true)
} else {
res, err = NewHttpClient(config.Config, req.Host).Do(req)
}
if res == nil {
res = &http.Response{
StatusCode: 0,
Header: make(http.Header),
Request: req,
Body: ioutil.NopCloser(bytes.NewBufferString("")),
}
}
if err != nil {
if errutil.IsAuthError(err) {
SetAuthType(req, res)
doHttpRequest(req, creds)
} else {
err = errutil.Error(err)
}
} else {
// TODO(sinbad) stop handling the response here, separate response processing to api package
err = handleResponse(res, creds)
}
if err != nil {
if res != nil {
SetErrorResponseContext(err, res)
} else {
setErrorRequestContext(err, req)
}
}
return res, err
}
// DoHttpRequest performs a single HTTP request
func DoHttpRequest(req *http.Request, useCreds bool) (*http.Response, error) {
var creds auth.Creds
if useCreds {
c, err := auth.GetCreds(req)
if err != nil {
return nil, err
}
creds = c
}
return doHttpRequest(req, creds)
}
// DoHttpRequestWithRedirects runs a HTTP request and responds to redirects
func DoHttpRequestWithRedirects(req *http.Request, via []*http.Request, useCreds bool) (*http.Response, error) {
var creds auth.Creds
if useCreds {
c, err := auth.GetCreds(req)
if err != nil {
return nil, err
}
creds = c
}
res, err := doHttpRequest(req, creds)
if err != nil {
return res, err
}
if res.StatusCode == 307 {
redirectTo := res.Header.Get("Location")
locurl, err := url.Parse(redirectTo)
if err == nil && !locurl.IsAbs() {
locurl = req.URL.ResolveReference(locurl)
redirectTo = locurl.String()
}
redirectedReq, err := NewHttpRequest(req.Method, redirectTo, nil)
if err != nil {
return res, errutil.Errorf(err, err.Error())
}
via = append(via, req)
// Avoid seeking and re-wrapping the CountingReadCloser, just get the "real" body
realBody := req.Body
if wrappedBody, ok := req.Body.(*CountingReadCloser); ok {
realBody = wrappedBody.ReadCloser
}
seeker, ok := realBody.(io.Seeker)
if !ok {
return res, errutil.Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.")
}
if _, err := seeker.Seek(0, 0); err != nil {
return res, errutil.Error(err)
}
redirectedReq.Body = realBody
redirectedReq.ContentLength = req.ContentLength
if err = CheckRedirect(redirectedReq, via); err != nil {
return res, errutil.Errorf(err, err.Error())
}
return DoHttpRequestWithRedirects(redirectedReq, via, useCreds)
}
return res, nil
}
// NewHttpRequest creates a template request, with the given headers & UserAgent supplied
func NewHttpRequest(method, rawurl string, header map[string]string) (*http.Request, error) {
req, err := http.NewRequest(method, rawurl, nil)
if err != nil {
return nil, err
}
for key, value := range header {
req.Header.Set(key, value)
}
req.Header.Set("User-Agent", UserAgent)
return req, nil
}
func SetAuthType(req *http.Request, res *http.Response) {
authType := GetAuthType(res)
operation := auth.GetOperationForRequest(req)
config.Config.SetAccess(operation, authType)
tracerx.Printf("api: http response indicates %q authentication. Resubmitting...", authType)
}
func GetAuthType(res *http.Response) string {
auth := res.Header.Get("Www-Authenticate")
if len(auth) < 1 {
auth = res.Header.Get("Lfs-Authenticate")
}
if strings.HasPrefix(strings.ToLower(auth), "ntlm") {
return "ntlm"
}
return "basic"
}

2
lfs/client_error_test.go → httputil/request_error_test.go
View File

@ -1,4 +1,4 @@
package lfs
package httputil
import (
"bytes"

127
httputil/response.go Normal file
View File

@ -0,0 +1,127 @@
package httputil
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"regexp"
"github.com/github/git-lfs/auth"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
)
var (
lfsMediaTypeRE = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
jsonMediaTypeRE = regexp.MustCompile(`\Aapplication/json(;|\z)`)
hiddenHeaders = map[string]bool{
"Authorization": true,
}
defaultErrors = map[int]string{
400: "Client error: %s",
401: "Authorization error: %s\nCheck that you have proper access to the repository",
403: "Authorization error: %s\nCheck that you have proper access to the repository",
404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
500: "Server error: %s",
}
)
// DecodeResponse attempts to decode the contents of the response as a JSON object
func DecodeResponse(res *http.Response, obj interface{}) error {
ctype := res.Header.Get("Content-Type")
if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) {
return nil
}
err := json.NewDecoder(res.Body).Decode(obj)
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
if err != nil {
return errutil.Errorf(err, "Unable to parse HTTP response for %s", TraceHttpReq(res.Request))
}
return nil
}
// GetDefaultError returns the default text for standard error codes (blank if none)
func GetDefaultError(code int) string {
if s, ok := defaultErrors[code]; ok {
return s
}
return ""
}
// Check the response from a HTTP request for problems
func handleResponse(res *http.Response, creds auth.Creds) error {
auth.SaveCredentials(creds, res)
if res.StatusCode < 400 {
return nil
}
defer func() {
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
}()
cliErr := &ClientError{}
err := DecodeResponse(res, cliErr)
if err == nil {
if len(cliErr.Message) == 0 {
err = defaultError(res)
} else {
err = errutil.Error(cliErr)
}
}
if res.StatusCode == 401 {
return errutil.NewAuthError(err)
}
if res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509 {
return errutil.NewFatalError(err)
}
return err
}
func defaultError(res *http.Response) error {
var msgFmt string
if f, ok := defaultErrors[res.StatusCode]; ok {
msgFmt = f
} else if res.StatusCode < 500 {
msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
} else {
msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
}
return errutil.Error(fmt.Errorf(msgFmt, res.Request.URL))
}
func SetErrorResponseContext(err error, res *http.Response) {
errutil.ErrorSetContext(err, "Status", res.Status)
setErrorHeaderContext(err, "Request", res.Header)
setErrorRequestContext(err, res.Request)
}
func setErrorRequestContext(err error, req *http.Request) {
errutil.ErrorSetContext(err, "Endpoint", config.Config.Endpoint(auth.GetOperationForRequest(req)).Url)
errutil.ErrorSetContext(err, "URL", TraceHttpReq(req))
setErrorHeaderContext(err, "Response", req.Header)
}
func setErrorHeaderContext(err error, prefix string, head http.Header) {
for key, _ := range head {
contextKey := fmt.Sprintf("%s:%s", prefix, key)
if _, skip := hiddenHeaders[key]; skip {
errutil.ErrorSetContext(err, contextKey, "--")
} else {
errutil.ErrorSetContext(err, contextKey, head.Get(key))
}
}
}

356
lfs/client.go
View File

@ -11,9 +11,7 @@ import (
"os"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"github.com/github/git-lfs/api"
"github.com/github/git-lfs/auth"
@ -29,39 +27,6 @@ const (
mediaType = "application/vnd.git-lfs+json; charset=utf-8"
)
var (
lfsMediaTypeRE = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
jsonMediaTypeRE = regexp.MustCompile(`\Aapplication/json(;|\z)`)
hiddenHeaders = map[string]bool{
"Authorization": true,
}
defaultErrors = map[int]string{
400: "Client error: %s",
401: "Authorization error: %s\nCheck that you have proper access to the repository",
403: "Authorization error: %s\nCheck that you have proper access to the repository",
404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
500: "Server error: %s",
}
)
type ClientError struct {
Message string `json:"message"`
DocumentationUrl string `json:"documentation_url,omitempty"`
RequestId string `json:"request_id,omitempty"`
}
func (e *ClientError) Error() string {
msg := e.Message
if len(e.DocumentationUrl) > 0 {
msg += "\nDocs: " + e.DocumentationUrl
}
if len(e.RequestId) > 0 {
msg += "\nRequest ID: " + e.RequestId
}
return msg
}
// Download will attempt to download the object with the given oid. The batched
// API will be used, but if the server does not implement the batch operations
// it will fall back to the legacy API.
@ -98,7 +63,7 @@ func DownloadLegacy(oid string) (io.ReadCloser, int64, error) {
return nil, 0, errutil.Error(err)
}
res, obj, err := doLegacyApiRequest(req)
res, obj, err := api.DoLegacyRequest(req)
if err != nil {
return nil, 0, err
}
@ -108,7 +73,7 @@ func DownloadLegacy(oid string) (io.ReadCloser, int64, error) {
return nil, 0, errutil.Error(err)
}
res, err = doHttpRequestWithCreds(req)
res, err = httputil.DoHttpRequest(req, true)
if err != nil {
return nil, 0, err
}
@ -127,7 +92,7 @@ func DownloadCheck(oid string) (*api.ObjectResource, error) {
return nil, errutil.Error(err)
}
res, obj, err := doLegacyApiRequest(req)
res, obj, err := api.DoLegacyRequest(req)
if err != nil {
return nil, err
}
@ -147,7 +112,7 @@ func DownloadObject(obj *api.ObjectResource) (io.ReadCloser, int64, error) {
return nil, 0, errutil.Error(err)
}
res, err := doHttpRequestWithCreds(req)
res, err := httputil.DoHttpRequest(req, true)
if err != nil {
return nil, 0, errutil.NewRetriableError(err)
}
@ -184,7 +149,7 @@ func Batch(objects []*api.ObjectResource, operation string) ([]*api.ObjectResour
tracerx.Printf("api: batch %d files", len(objects))
res, objs, err := doApiBatchRequest(req)
res, objs, err := api.DoBatchRequest(req)
if err != nil {
@ -197,7 +162,7 @@ func Batch(objects []*api.ObjectResource, operation string) ([]*api.ObjectResour
}
if errutil.IsAuthError(err) {
setAuthType(req, res)
httputil.SetAuthType(req, res)
return Batch(objects, operation)
}
@ -248,11 +213,11 @@ func UploadCheck(oidPath string) (*api.ObjectResource, error) {
req.Body = &byteCloser{bytes.NewReader(by)}
tracerx.Printf("api: uploading (%s)", oid)
res, obj, err := doLegacyApiRequest(req)
res, obj, err := api.DoLegacyRequest(req)
if err != nil {
if errutil.IsAuthError(err) {
setAuthType(req, res)
httputil.SetAuthType(req, res)
return UploadCheck(oidPath)
}
@ -310,7 +275,7 @@ func UploadObject(o *api.ObjectResource, cb progress.CopyCallback) error {
req.ContentLength = o.Size
req.Body = ioutil.NopCloser(reader)
res, err := doHttpRequestWithCreds(req)
res, err := httputil.DoHttpRequest(req, true)
if err != nil {
return errutil.NewRetriableError(err)
}
@ -347,7 +312,7 @@ func UploadObject(o *api.ObjectResource, cb progress.CopyCallback) error {
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
req.ContentLength = int64(len(by))
req.Body = ioutil.NopCloser(bytes.NewReader(by))
res, err = doAPIRequest(req, true)
res, err = api.DoRequest(req, true)
if err != nil {
return err
}
@ -359,234 +324,6 @@ func UploadObject(o *api.ObjectResource, cb progress.CopyCallback) error {
return err
}
// doLegacyApiRequest runs the request to the LFS legacy API.
func doLegacyApiRequest(req *http.Request) (*http.Response, *api.ObjectResource, error) {
via := make([]*http.Request, 0, 4)
res, err := doApiRequestWithRedirects(req, via, true)
if err != nil {
return res, nil, err
}
obj := &api.ObjectResource{}
err = decodeApiResponse(res, obj)
if err != nil {
setErrorResponseContext(err, res)
return nil, nil, err
}
return res, obj, nil
}
// doApiBatchRequest runs the request to the LFS batch API. If the API returns a
// 401, the repo will be marked as having private access and the request will be
// re-run. When the repo is marked as having private access, credentials will
// be retrieved.
func doApiBatchRequest(req *http.Request) (*http.Response, []*api.ObjectResource, error) {
res, err := doAPIRequest(req, config.Config.PrivateAccess(httputil.GetOperationForRequest(req)))
if err != nil {
if res != nil && res.StatusCode == 401 {
return res, nil, errutil.NewAuthError(err)
}
return res, nil, err
}
var objs map[string][]*api.ObjectResource
err = decodeApiResponse(res, &objs)
if err != nil {
setErrorResponseContext(err, res)
}
return res, objs["objects"], err
}
// doHttpRequestWithCreds performs doHttpRequest with creds added
func doHttpRequestWithCreds(req *http.Request) (*http.Response, error) {
creds, err := auth.GetCreds(req)
if err != nil {
return nil, err
}
return doHttpRequest(req, creds)
}
// doAPIRequest runs the request to the LFS API, without parsing the response
// body. If the API returns a 401, the repo will be marked as having private
// access and the request will be re-run. When the repo is marked as having
// private access, credentials will be retrieved.
func doAPIRequest(req *http.Request, useCreds bool) (*http.Response, error) {
via := make([]*http.Request, 0, 4)
return doApiRequestWithRedirects(req, via, useCreds)
}
// doHttpRequest runs the given HTTP request. LFS or Storage API requests should
// use doApiBatchRequest() or doHttpRequestWithCreds() instead.
func doHttpRequest(req *http.Request, creds auth.Creds) (*http.Response, error) {
var (
res *http.Response
err error
)
if config.Config.NtlmAccess(httputil.GetOperationForRequest(req)) {
res, err = auth.DoNTLMRequest(req, true)
} else {
res, err = httputil.NewHttpClient(config.Config, req.Host).Do(req)
}
if res == nil {
res = &http.Response{
StatusCode: 0,
Header: make(http.Header),
Request: req,
Body: ioutil.NopCloser(bytes.NewBufferString("")),
}
}
if err != nil {
if errutil.IsAuthError(err) {
setAuthType(req, res)
doHttpRequest(req, creds)
} else {
err = errutil.Error(err)
}
} else {
err = handleResponse(res, creds)
}
if err != nil {
if res != nil {
setErrorResponseContext(err, res)
} else {
setErrorRequestContext(err, req)
}
}
return res, err
}
func doApiRequestWithRedirects(req *http.Request, via []*http.Request, useCreds bool) (*http.Response, error) {
var creds auth.Creds
if useCreds {
c, err := auth.GetCreds(req)
if err != nil {
return nil, err
}
creds = c
}
res, err := doHttpRequest(req, creds)
if err != nil {
return res, err
}
if res.StatusCode == 307 {
redirectTo := res.Header.Get("Location")
locurl, err := url.Parse(redirectTo)
if err == nil && !locurl.IsAbs() {
locurl = req.URL.ResolveReference(locurl)
redirectTo = locurl.String()
}
redirectedReq, err := newClientRequest(req.Method, redirectTo, nil)
if err != nil {
return res, errutil.Errorf(err, err.Error())
}
via = append(via, req)
// Avoid seeking and re-wrapping the CountingReadCloser, just get the "real" body
realBody := req.Body
if wrappedBody, ok := req.Body.(*httputil.CountingReadCloser); ok {
realBody = wrappedBody.ReadCloser
}
seeker, ok := realBody.(io.Seeker)
if !ok {
return res, errutil.Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.")
}
if _, err := seeker.Seek(0, 0); err != nil {
return res, errutil.Error(err)
}
redirectedReq.Body = realBody
redirectedReq.ContentLength = req.ContentLength
if err = httputil.CheckRedirect(redirectedReq, via); err != nil {
return res, errutil.Errorf(err, err.Error())
}
return doApiRequestWithRedirects(redirectedReq, via, useCreds)
}
return res, nil
}
func handleResponse(res *http.Response, creds auth.Creds) error {
auth.SaveCredentials(creds, res)
if res.StatusCode < 400 {
return nil
}
defer func() {
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
}()
cliErr := &ClientError{}
err := decodeApiResponse(res, cliErr)
if err == nil {
if len(cliErr.Message) == 0 {
err = defaultError(res)
} else {
err = errutil.Error(cliErr)
}
}
if res.StatusCode == 401 {
return errutil.NewAuthError(err)
}
if res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509 {
return errutil.NewFatalError(err)
}
return err
}
func decodeApiResponse(res *http.Response, obj interface{}) error {
ctype := res.Header.Get("Content-Type")
if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) {
return nil
}
err := json.NewDecoder(res.Body).Decode(obj)
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
if err != nil {
return errutil.Errorf(err, "Unable to parse HTTP response for %s", httputil.TraceHttpReq(res.Request))
}
return nil
}
func defaultError(res *http.Response) error {
var msgFmt string
if f, ok := defaultErrors[res.StatusCode]; ok {
msgFmt = f
} else if res.StatusCode < 500 {
msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
} else {
msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
}
return errutil.Error(fmt.Errorf(msgFmt, res.Request.URL))
}
func newApiRequest(method, oid string) (*http.Request, error) {
objectOid := oid
operation := "download"
@ -615,7 +352,7 @@ func newApiRequest(method, oid string) (*http.Request, error) {
return nil, err
}
req, err := newClientRequest(method, u.String(), res.Header)
req, err := httputil.NewHttpRequest(method, u.String(), res.Header)
if err != nil {
return nil, err
}
@ -624,21 +361,6 @@ func newApiRequest(method, oid string) (*http.Request, error) {
return req, nil
}
func newClientRequest(method, rawurl string, header map[string]string) (*http.Request, error) {
req, err := http.NewRequest(method, rawurl, nil)
if err != nil {
return nil, err
}
for key, value := range header {
req.Header.Set(key, value)
}
req.Header.Set("User-Agent", httputil.UserAgent)
return req, nil
}
func newBatchApiRequest(operation string) (*http.Request, error) {
endpoint := config.Config.Endpoint(operation)
@ -659,7 +381,7 @@ func newBatchApiRequest(operation string) (*http.Request, error) {
return nil, err
}
req, err := newBatchClientRequest("POST", u.String())
req, err := httputil.NewHttpRequest("POST", u.String(), nil)
if err != nil {
return nil, err
}
@ -674,60 +396,6 @@ func newBatchApiRequest(operation string) (*http.Request, error) {
return req, nil
}
func newBatchClientRequest(method, rawurl string) (*http.Request, error) {
req, err := http.NewRequest(method, rawurl, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", httputil.UserAgent)
return req, nil
}
func setAuthType(req *http.Request, res *http.Response) {
authType := getAuthType(res)
operation := httputil.GetOperationForRequest(req)
config.Config.SetAccess(operation, authType)
tracerx.Printf("api: http response indicates %q authentication. Resubmitting...", authType)
}
func getAuthType(res *http.Response) string {
auth := res.Header.Get("Www-Authenticate")
if len(auth) < 1 {
auth = res.Header.Get("Lfs-Authenticate")
}
if strings.HasPrefix(strings.ToLower(auth), "ntlm") {
return "ntlm"
}
return "basic"
}
func setErrorResponseContext(err error, res *http.Response) {
errutil.ErrorSetContext(err, "Status", res.Status)
setErrorHeaderContext(err, "Request", res.Header)
setErrorRequestContext(err, res.Request)
}
func setErrorRequestContext(err error, req *http.Request) {
errutil.ErrorSetContext(err, "Endpoint", config.Config.Endpoint(httputil.GetOperationForRequest(req)).Url)
errutil.ErrorSetContext(err, "URL", httputil.TraceHttpReq(req))
setErrorHeaderContext(err, "Response", req.Header)
}
func setErrorHeaderContext(err error, prefix string, head http.Header) {
for key, _ := range head {
contextKey := fmt.Sprintf("%s:%s", prefix, key)
if _, skip := hiddenHeaders[key]; skip {
errutil.ErrorSetContext(err, contextKey, "--")
} else {
errutil.ErrorSetContext(err, contextKey, head.Get(key))
}
}
}
func ObjectUrl(endpoint config.Endpoint, oid string) (*url.URL, error) {
u, err := url.Parse(endpoint.Url)
if err != nil {

5
lfs/download_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/github/git-lfs/api"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/httputil"
)
func TestSuccessfulDownload(t *testing.T) {
@ -650,7 +651,7 @@ func TestDownloadAPIError(t *testing.T) {
return
}
if err.Error() != fmt.Sprintf(defaultErrors[404], server.URL+"/media/objects/oid") {
if err.Error() != fmt.Sprintf(httputil.GetDefaultError(404), server.URL+"/media/objects/oid") {
t.Fatalf("Unexpected error: %s", err.Error())
}
@ -727,7 +728,7 @@ func TestDownloadStorageError(t *testing.T) {
t.Fatal("should panic")
}
if err.Error() != fmt.Sprintf(defaultErrors[500], server.URL+"/download") {
if err.Error() != fmt.Sprintf(httputil.GetDefaultError(500), server.URL+"/download") {
t.Fatalf("Unexpected error: %s", err.Error())
}

7
lfs/upload_test.go
View File

@ -15,6 +15,7 @@ import (
"github.com/github/git-lfs/api"
"github.com/github/git-lfs/config"
"github.com/github/git-lfs/errutil"
"github.com/github/git-lfs/httputil"
)
func TestExistingUpload(t *testing.T) {
@ -662,7 +663,7 @@ func TestUploadApiError(t *testing.T) {
return
}
if err.Error() != fmt.Sprintf(defaultErrors[404], server.URL+"/media/objects") {
if err.Error() != fmt.Sprintf(httputil.GetDefaultError(404), server.URL+"/media/objects") {
t.Fatalf("Unexpected error: %s", err.Error())
}
@ -781,7 +782,7 @@ func TestUploadStorageError(t *testing.T) {
t.Fatal("should not panic")
}
if err.Error() != fmt.Sprintf(defaultErrors[404], server.URL+"/upload") {
if err.Error() != fmt.Sprintf(httputil.GetDefaultError(404), server.URL+"/upload") {
t.Fatalf("Unexpected error: %s", err.Error())
}
@ -937,7 +938,7 @@ func TestUploadVerifyError(t *testing.T) {
t.Fatal("should not panic")
}
if err.Error() != fmt.Sprintf(defaultErrors[404], server.URL+"/verify") {
if err.Error() != fmt.Sprintf(httputil.GetDefaultError(404), server.URL+"/verify") {
t.Fatalf("Unexpected error: %s", err.Error())
}