diff --git a/docs/api.md b/docs/api.md index 05a12737..9acaff73 100644 --- a/docs/api.md +++ b/docs/api.md @@ -7,55 +7,95 @@ Git repositories that use Hawser will specify a URI endpoint. See the Use that endpoint as a base, and append the following relative paths to upload and download from the Hawser server. -## GET /objects/{oid} +All requests should send an Accept header of `application/vnd.hawser+json`. +This may change in the future as the API evolves. -This gets either the object content, or the object's meta data. The OID is the -value from the object pointer. +## API Responses -### Getting the content +This specification defines what status codes that API can return. Look at each +individual API method for more details. Some of the specific status codes may +trigger specific error messages from the client. -To download the object content, send an Accept header of `application/vnd.hawser`. -The server returns the raw content back with a `Content-Type` of -`application/octet-stream`. +* 200 - The request completed successfully. +* 202 - An upload request has been accepted. Clients should follow hypermedia +links to actually upload the content. +* 400 - General error with the client's request. Invalid JSON formatting, for +example. +* 401 - The authentication credentials are incorrect. +* 403 - The requesting user has access to see the repository, but not to push +changes to it. +* 404 - Either the user does not have access to see the repository, or the +repository or requested object does not exist. + +The following status codes can optionally be returned from the API, depending on +the server implementation. + +* 406 - The Accept header is invalid. It should be `application/vnd.hawser+json`. +* 429 - The user has hit a rate limit with the server. Though the API does not +specify any rate limits, implementors are encouraged to set some for +availability reasons. +* 501 - The server has not implemented the current method. Reserved for future +use. +* 509 - Returned if the bandwidth limit for the user or repository has been +exceeded. The API does not specify any bandwidth limit, but implementors may +track usage. + +Some server errors may trigger the client to retry requests, such as 500, 502, +503, and 504. + +If the server returns a JSON error object, the client can display this message +to users. ``` -> GET https://hawser-server.com/objects/{oid} HTTP/1.1 -> Accept: application/octet-stream -> Authorization: Basic ... (if authentication is needed) +> GET https://hawser-server.com/objects/{OID} HTTP/1.1 +> Accept: application/vnd.hawser+json > < HTTP/1.1 200 OK -< Content-Type: application/octet-stream +< Content-Type: application/vnd.hawser+json < -< {binary contents} +< { +< "message": "Bad credentials", +< "documentation_url": "https://hawser-server.com/docs/errors", +< "request_id": "123" +< } ``` -The server can also redirect to another location. This is useful in cases where -you do not want to render user content on a domain with important cookies. -Request headers like `Range` or `Accept` should be passed through. The -`Authorization` header must _not_ be passed through if the location's host or -scheme differs from the original request uri. +The `documentation_url` and `request_id` properties are optional. If given, +they are displayed to the user. -``` -> GET https://hawser-server.com/objects/{oid} HTTP/1.1 -> Accept: application/vnd.hawser -> Authorization: Basic ... (if authentication is needed) -> -< HTTP/1.1 302 Found -< Location: https://storage-server.com/{oid} -< -< {binary contents} -``` +## Hypermedia -### Responses +The Hawser API uses hypermedia hints to instruct the client what to do next. +These links are included in a `_links` property. Possible relations for objects +include: -* 200 - The object contents or meta data is in the response. -* 302 - Temporary redirect to a new location. -* 404 - The user does not have access to the object, or it does not exist. +* `self` - This points to the object's canonical API URL. +* `download` - Follow this link with a GET and the optional header values to +download the object content. +* `upload` - Upload the object content to this link with a PUT. +* `verify` - Optional link for the client to POST after an upload. If +included, the client assumes this step is required after uploading an object. +See the "Verification" section below for more. -### Getting meta data. +Link relations specify the `href`, and optionally a collection of header values +to set for the request. These are optional, and depend on the backing object +store that the Hawser API is using. -You can also request just the JSON meta data with an `Accept` header of -`application/vnd.hawser+json`. Here's an example successful request: +The Hawser client will automatically send the same credentials to the followed +link relation as Basic Authentication IF: + +* The url scheme, host, and port all match the Hawser API endpoint's. +* The link relation does not specify an Authorization header. + +If the host name is different, the Hawser API needs to send enough information +through the href query or header values to authenticate the request. + +The Hawser client expects a 200 or 201 response from these hypermedia requests. +Any other response code is treated as an error. + +## GET /objects/{oid} + +This gets the object's meta data. The OID is the value from the object pointer. ``` > GET https://hawser-server.com/objects/{OID} HTTP/1.1 @@ -69,6 +109,9 @@ You can also request just the JSON meta data with an `Accept` header of < "oid": "the-sha-256-signature", < "size": 123456, < "_links": { +< "self": { +< "href": "https://hawser-server.com/objects/OID", +< }, < "download": { < "href": "https://some-download.com", < "header": { @@ -80,10 +123,8 @@ You can also request just the JSON meta data with an `Accept` header of ``` The `oid` and `size` properties are required. A hypermedia `_links` section is -included with a `download` link relation, which describes how to download the -object content. If the GET request to download an object (with `Accept: -application/octet-stream`) redirects somewhere else, a similar URL should be -used with the `download` relation. +included with a `download` link relation. Clients can follow this link to +access the object content. See the "Hypermedia" section above for more. Here's a sample response for a request with an authorization error: @@ -100,39 +141,11 @@ Here's a sample response for a request with an authorization error: < } ``` -There are what the HTTP status codes mean: +### Responses -* 200 - The user is able to read the object. -* 404 - The repository does not exist for the user, or the user does not have -access to it. - -## OPTIONS /objects/{oid} - -This is a pre-flight request to verify credentials before sending the file -contents. Note: The `OPTIONS` method is only supported in pre-1.0 Hawser -clients. After 1.0, clients should use the `GET` with the -`application/vnd.hawser+json` Accept header. - -Here's an example successful request: - -``` -> OPTIONS https://hawser-server.com/objects/{OID} HTTP/1.1 -> Accept: application/vnd.hawser+json -> Authorization: Basic ... (if authentication is needed) -> -< HTTP/1.1 200 OK - -(no response body) -``` - -There are what the HTTP status codes mean: - -* 200 - The user is able to read the object. -* 204 - The user is able to PUT the object to the same URL. -* 403 - The user has **read**, but not **write** access. -* 404 - The repository does not exist for the user. -* 405 - OPTIONS not supported, use a GET request with a `application/vnd.hawser+json` -Accept header. +* 200 - The object exists and the user has access to download it. +* 401 - The authentication credentials are incorrect. +* 404 - The user does not have access to the object, or it does not exist. ## POST /objects @@ -174,44 +187,23 @@ and size of the object to upload. A response can include one of multiple link relations, each with an `href` property and an optional `header` property. -* `upload` - This relation describes how to upload the object. +* `upload` - This relation describes how to upload the object. Expect this with +a 202 status. * `verify` - The server can specify a URL for the client to hit after -successfully uploading an object. -* `download` - This relation describes how to download the object content. +successfully uploading an object. This is an optional relation for a 202 +status. +* `download` - This relation describes how to download the object content. This +only appears on a 200 status. ### Responses * 200 - The object already exists. Don't bother re-uploading. -* 202 - The object is ready to be uploaded.Follow the "upload" and optional +* 202 - The object is ready to be uploaded. Follow the "upload" and optional "verify" links. +* 401 - The authentication credentials are incorrect. * 403 - The user has **read**, but not **write** access. * 404 - The repository does not exist for the user. -## PUT /objects/{oid} - -This writes the object contents to the Git Media server. - -``` -> PUT https://hawser-server.com/objects/{oid} HTTP/1.1 -> Accept: application/vnd.hawser -> Content-Type: application/octet-stream -> Authorization: Basic ... -> Content-Length: 123 -> -> {binary contents} -> -< HTTP/1.1 200 OK -``` - -### Responses - -* 200 - The object already exists. -* 201 - The object was uploaded successfully. -* 403 - The user has **read**, but not **write** access. -* 404 - The repository does not exist for the user. -* 405 - PUT method is not allowed. Use an OPTIONS or GET pre-flight request to -get the current URL to send a file. - ## Verification When Hawser clients issue a POST request to initiate an object upload, the diff --git a/hawser/client.go b/hawser/client.go index eee4f2d6..59cce9b7 100644 --- a/hawser/client.go +++ b/hawser/client.go @@ -38,7 +38,7 @@ func Download(oidPath string) (io.ReadCloser, int64, *WrappedError) { } req.Header.Set("Accept", gitMediaType) - res, wErr := doRequest(req, creds) + res, wErr := doHTTPWithCreds(req, creds) if wErr != nil { return nil, 0, wErr @@ -190,11 +190,10 @@ func callOptions(filehash string) (int, *WrappedError) { return 0, Errorf(err, "Unable to build OPTIONS request for %s", oid) } - res, wErr := doRequest(req, creds) + res, wErr := doHTTPWithCreds(req, creds) if wErr != nil { return 0, wErr } - tracerx.Printf("api_options_status: %d", res.StatusCode) return res.StatusCode, nil } @@ -240,8 +239,7 @@ func callPut(filehash, filename string, cb CopyCallback) *WrappedError { fmt.Printf("Sending %s\n", filename) tracerx.Printf("api_put: %s %s", oid, filename) - res, wErr := doRequest(req, creds) - tracerx.Printf("api_put_status: %d", res.StatusCode) + _, wErr := doHTTPWithCreds(req, creds) return wErr } @@ -296,7 +294,6 @@ func callExternalPut(filehash, filename string, obj *objectResource, cb CopyCall if err != nil { return Errorf(err, "Error attempting to PUT %s", filename) } - tracerx.Printf("external_put_status: %d", res.StatusCode) saveCredentials(creds, res) // Run the verify callback @@ -323,7 +320,6 @@ func callExternalPut(filehash, filename string, obj *objectResource, cb CopyCall if err != nil { return Errorf(err, "Error attempting to verify %s", filename) } - tracerx.Printf("verify_status: %d", verifyRes.StatusCode) saveCredentials(verifyCreds, verifyRes) return nil @@ -354,11 +350,10 @@ func callPost(filehash, filename string) (*objectResource, int, *WrappedError) { req.Header.Set("Accept", gitMediaMetaType) tracerx.Printf("api_post: %s %s", oid, filename) - res, wErr := doRequest(req, creds) + res, wErr := doHTTPWithCreds(req, creds) if wErr != nil { return nil, 0, wErr } - tracerx.Printf("api_post_status: %d", res.StatusCode) if res.StatusCode == 202 { obj := &objectResource{} @@ -404,7 +399,9 @@ func validateMediaHeader(contentType string, reader io.Reader) (bool, int, *Wrap return true, headerSize, nil } -func doRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) { +// Wraps DoHTTP(), and saves or removes credentials from the git credential +// store based on the response. +func doHTTPWithCreds(req *http.Request, creds Creds) (*http.Response, *WrappedError) { res, err := DoHTTP(Config, req) var wErr *WrappedError @@ -475,7 +472,7 @@ func saveCredentials(creds Creds, res *http.Response) { return } - if res.StatusCode < 405 { + if res.StatusCode < 404 { execCreds(creds, "reject") } } @@ -537,10 +534,18 @@ func setRequestHeaders(req *http.Request) (Creds, error) { } type ClientError struct { - Message string `json:"message"` - RequestId string `json:"request_id,omitempty"` + Message string `json:"message"` + DocumentationUrl string `json:"documentation_url,omitempty"` + RequestId string `json:"request_id,omitempty"` } func (e *ClientError) Error() string { - return e.Message + msg := e.Message + if len(e.DocumentationUrl) > 0 { + msg += "\nDocs: " + e.DocumentationUrl + } + if len(e.RequestId) > 0 { + msg += "\nRequest ID: " + e.RequestId + } + return msg } diff --git a/hawser/config.go b/hawser/config.go index a3c97b4a..2f753546 100644 --- a/hawser/config.go +++ b/hawser/config.go @@ -17,15 +17,24 @@ type Configuration struct { remotes []string httpClient *http.Client redirectingHttpClient *http.Client + isTracingHttp bool } var ( - Config = &Configuration{CurrentRemote: defaultRemote} + Config = NewConfig() RedirectError = fmt.Errorf("Unexpected redirection") httpPrefixRe = regexp.MustCompile("\\Ahttps?://") defaultRemote = "origin" ) +func NewConfig() *Configuration { + c := &Configuration{ + CurrentRemote: defaultRemote, + isTracingHttp: len(os.Getenv("GIT_CURL_VERBOSE")) > 0, + } + return c +} + func (c *Configuration) Endpoint() string { if url, ok := c.GitConfig("hawser.url"); ok { return url @@ -69,24 +78,18 @@ func (c *Configuration) RemoteEndpoint(remote string) string { } func (c *Configuration) Remotes() []string { - if c.remotes == nil { - c.loadGitConfig() - } + c.loadGitConfig() return c.remotes } func (c *Configuration) GitConfig(key string) (string, bool) { - if c.gitConfig == nil { - c.loadGitConfig() - } + c.loadGitConfig() value, ok := c.gitConfig[strings.ToLower(key)] return value, ok } func (c *Configuration) SetConfig(key, value string) { - if c.gitConfig == nil { - c.loadGitConfig() - } + c.loadGitConfig() c.gitConfig[key] = value } @@ -107,6 +110,10 @@ type AltConfig struct { } func (c *Configuration) loadGitConfig() { + if c.gitConfig != nil { + return + } + uniqRemotes := make(map[string]bool) c.gitConfig = make(map[string]string) @@ -148,10 +155,3 @@ func (c *Configuration) loadGitConfig() { c.remotes = append(c.remotes, remote) } } - -func configFileExists(filename string) bool { - if _, err := os.Stat(filename); err == nil { - return true - } - return false -} diff --git a/hawser/http.go b/hawser/http.go index 5967f2f0..fb6948de 100644 --- a/hawser/http.go +++ b/hawser/http.go @@ -2,17 +2,36 @@ package hawser import ( "crypto/tls" + "fmt" + "github.com/rubyist/tracerx" + "io" "net/http" "os" + "strings" ) func DoHTTP(c *Configuration, req *http.Request) (*http.Response, error) { + var res *http.Response + var err error + + var counter *countingBody + if req.Body != nil { + counter = newCountingBody(req.Body) + req.Body = counter + } + + traceHttpRequest(c, req) + switch req.Method { case "GET", "HEAD": - return c.RedirectingHttpClient().Do(req) + res, err = c.RedirectingHttpClient().Do(req) default: - return c.HttpClient().Do(req) + res, err = c.HttpClient().Do(req) } + + traceHttpResponse(c, res, counter) + + return res, err } func (c *Configuration) HttpClient() *http.Client { @@ -29,18 +48,97 @@ func (c *Configuration) HttpClient() *http.Client { func (c *Configuration) RedirectingHttpClient() *http.Client { if c.redirectingHttpClient == nil { - c.redirectingHttpClient = &http.Client{ - Transport: httpTransportFor(c), + tr := &http.Transport{} + sslVerify, _ := c.GitConfig("http.sslverify") + if sslVerify == "false" || len(os.Getenv("GIT_SSL_NO_VERIFY")) > 0 { + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} } + c.redirectingHttpClient = &http.Client{Transport: tr} } return c.redirectingHttpClient } -func httpTransportFor(c *Configuration) *http.Transport { - tr := &http.Transport{} - sslVerify, _ := c.GitConfig("http.sslverify") - if len(os.Getenv("GIT_SSL_NO_VERIFY")) > 0 || sslVerify == "false" { - tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} +var tracedTypes = []string{"json", "text", "xml", "html"} + +func traceHttpRequest(c *Configuration, req *http.Request) { + tracerx.Printf("HTTP: %s %s", req.Method, req.URL.String()) + + if c.isTracingHttp == false { + return + } + + fmt.Fprintf(os.Stderr, "> %s %s %s\n", req.Method, req.URL.RequestURI(), req.Proto) + for key, _ := range req.Header { + fmt.Fprintf(os.Stderr, "> %s: %s\n", key, req.Header.Get(key)) } - return tr +} + +func traceHttpResponse(c *Configuration, res *http.Response, counter *countingBody) { + tracerx.Printf("HTTP: %d", res.StatusCode) + + if c.isTracingHttp == false { + return + } + + if counter != nil { + fmt.Fprintf(os.Stderr, "* upload sent off: %d bytes\n", counter.Size) + } + fmt.Fprintf(os.Stderr, "\n") + + fmt.Fprintf(os.Stderr, "< %s %s\n", res.Proto, res.Status) + for key, _ := range res.Header { + fmt.Fprintf(os.Stderr, "< %s: %s\n", key, res.Header.Get(key)) + } + + traceBody := false + ctype := strings.ToLower(strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]) + for _, tracedType := range tracedTypes { + if strings.Contains(ctype, tracedType) { + traceBody = true + } + } + + if traceBody { + fmt.Fprintf(os.Stderr, "\n") + res.Body = newTracedBody(res.Body) + } + + fmt.Fprintf(os.Stderr, "\n") +} + +type countingBody struct { + body io.ReadCloser + Size int64 +} + +func (r *countingBody) Read(p []byte) (int, error) { + n, err := r.body.Read(p) + r.Size += int64(n) + return n, err +} + +func (r *countingBody) Close() error { + return r.body.Close() +} + +func newCountingBody(body io.ReadCloser) *countingBody { + return &countingBody{body, 0} +} + +type tracedBody struct { + body io.ReadCloser +} + +func (r *tracedBody) Read(p []byte) (int, error) { + n, err := r.body.Read(p) + fmt.Fprintf(os.Stderr, "%s\n", string(p[0:n])) + return n, err +} + +func (r *tracedBody) Close() error { + return r.body.Close() +} + +func newTracedBody(body io.ReadCloser) *tracedBody { + return &tracedBody{body} }