Merge branch 'master' into ensure-cleaned-objects
This commit is contained in:
commit
5d14e02d8b
190
docs/api.md
190
docs/api.md
@ -7,55 +7,95 @@ Git repositories that use Hawser will specify a URI endpoint. See the
|
||||
Use that endpoint as a base, and append the following relative paths to upload
|
||||
and download from the Hawser server.
|
||||
|
||||
## GET /objects/{oid}
|
||||
All requests should send an Accept header of `application/vnd.hawser+json`.
|
||||
This may change in the future as the API evolves.
|
||||
|
||||
This gets either the object content, or the object's meta data. The OID is the
|
||||
value from the object pointer.
|
||||
## API Responses
|
||||
|
||||
### Getting the content
|
||||
This specification defines what status codes that API can return. Look at each
|
||||
individual API method for more details. Some of the specific status codes may
|
||||
trigger specific error messages from the client.
|
||||
|
||||
To download the object content, send an Accept header of `application/vnd.hawser`.
|
||||
The server returns the raw content back with a `Content-Type` of
|
||||
`application/octet-stream`.
|
||||
* 200 - The request completed successfully.
|
||||
* 202 - An upload request has been accepted. Clients should follow hypermedia
|
||||
links to actually upload the content.
|
||||
* 400 - General error with the client's request. Invalid JSON formatting, for
|
||||
example.
|
||||
* 401 - The authentication credentials are incorrect.
|
||||
* 403 - The requesting user has access to see the repository, but not to push
|
||||
changes to it.
|
||||
* 404 - Either the user does not have access to see the repository, or the
|
||||
repository or requested object does not exist.
|
||||
|
||||
The following status codes can optionally be returned from the API, depending on
|
||||
the server implementation.
|
||||
|
||||
* 406 - The Accept header is invalid. It should be `application/vnd.hawser+json`.
|
||||
* 429 - The user has hit a rate limit with the server. Though the API does not
|
||||
specify any rate limits, implementors are encouraged to set some for
|
||||
availability reasons.
|
||||
* 501 - The server has not implemented the current method. Reserved for future
|
||||
use.
|
||||
* 509 - Returned if the bandwidth limit for the user or repository has been
|
||||
exceeded. The API does not specify any bandwidth limit, but implementors may
|
||||
track usage.
|
||||
|
||||
Some server errors may trigger the client to retry requests, such as 500, 502,
|
||||
503, and 504.
|
||||
|
||||
If the server returns a JSON error object, the client can display this message
|
||||
to users.
|
||||
|
||||
```
|
||||
> GET https://hawser-server.com/objects/{oid} HTTP/1.1
|
||||
> Accept: application/octet-stream
|
||||
> Authorization: Basic ... (if authentication is needed)
|
||||
> GET https://hawser-server.com/objects/{OID} HTTP/1.1
|
||||
> Accept: application/vnd.hawser+json
|
||||
>
|
||||
< HTTP/1.1 200 OK
|
||||
< Content-Type: application/octet-stream
|
||||
< Content-Type: application/vnd.hawser+json
|
||||
<
|
||||
< {binary contents}
|
||||
< {
|
||||
< "message": "Bad credentials",
|
||||
< "documentation_url": "https://hawser-server.com/docs/errors",
|
||||
< "request_id": "123"
|
||||
< }
|
||||
```
|
||||
|
||||
The server can also redirect to another location. This is useful in cases where
|
||||
you do not want to render user content on a domain with important cookies.
|
||||
Request headers like `Range` or `Accept` should be passed through. The
|
||||
`Authorization` header must _not_ be passed through if the location's host or
|
||||
scheme differs from the original request uri.
|
||||
The `documentation_url` and `request_id` properties are optional. If given,
|
||||
they are displayed to the user.
|
||||
|
||||
```
|
||||
> GET https://hawser-server.com/objects/{oid} HTTP/1.1
|
||||
> Accept: application/vnd.hawser
|
||||
> Authorization: Basic ... (if authentication is needed)
|
||||
>
|
||||
< HTTP/1.1 302 Found
|
||||
< Location: https://storage-server.com/{oid}
|
||||
<
|
||||
< {binary contents}
|
||||
```
|
||||
## Hypermedia
|
||||
|
||||
### Responses
|
||||
The Hawser API uses hypermedia hints to instruct the client what to do next.
|
||||
These links are included in a `_links` property. Possible relations for objects
|
||||
include:
|
||||
|
||||
* 200 - The object contents or meta data is in the response.
|
||||
* 302 - Temporary redirect to a new location.
|
||||
* 404 - The user does not have access to the object, or it does not exist.
|
||||
* `self` - This points to the object's canonical API URL.
|
||||
* `download` - Follow this link with a GET and the optional header values to
|
||||
download the object content.
|
||||
* `upload` - Upload the object content to this link with a PUT.
|
||||
* `verify` - Optional link for the client to POST after an upload. If
|
||||
included, the client assumes this step is required after uploading an object.
|
||||
See the "Verification" section below for more.
|
||||
|
||||
### Getting meta data.
|
||||
Link relations specify the `href`, and optionally a collection of header values
|
||||
to set for the request. These are optional, and depend on the backing object
|
||||
store that the Hawser API is using.
|
||||
|
||||
You can also request just the JSON meta data with an `Accept` header of
|
||||
`application/vnd.hawser+json`. Here's an example successful request:
|
||||
The Hawser client will automatically send the same credentials to the followed
|
||||
link relation as Basic Authentication IF:
|
||||
|
||||
* The url scheme, host, and port all match the Hawser API endpoint's.
|
||||
* The link relation does not specify an Authorization header.
|
||||
|
||||
If the host name is different, the Hawser API needs to send enough information
|
||||
through the href query or header values to authenticate the request.
|
||||
|
||||
The Hawser client expects a 200 or 201 response from these hypermedia requests.
|
||||
Any other response code is treated as an error.
|
||||
|
||||
## GET /objects/{oid}
|
||||
|
||||
This gets the object's meta data. The OID is the value from the object pointer.
|
||||
|
||||
```
|
||||
> GET https://hawser-server.com/objects/{OID} HTTP/1.1
|
||||
@ -69,6 +109,9 @@ You can also request just the JSON meta data with an `Accept` header of
|
||||
< "oid": "the-sha-256-signature",
|
||||
< "size": 123456,
|
||||
< "_links": {
|
||||
< "self": {
|
||||
< "href": "https://hawser-server.com/objects/OID",
|
||||
< },
|
||||
< "download": {
|
||||
< "href": "https://some-download.com",
|
||||
< "header": {
|
||||
@ -80,10 +123,8 @@ You can also request just the JSON meta data with an `Accept` header of
|
||||
```
|
||||
|
||||
The `oid` and `size` properties are required. A hypermedia `_links` section is
|
||||
included with a `download` link relation, which describes how to download the
|
||||
object content. If the GET request to download an object (with `Accept:
|
||||
application/octet-stream`) redirects somewhere else, a similar URL should be
|
||||
used with the `download` relation.
|
||||
included with a `download` link relation. Clients can follow this link to
|
||||
access the object content. See the "Hypermedia" section above for more.
|
||||
|
||||
Here's a sample response for a request with an authorization error:
|
||||
|
||||
@ -100,39 +141,11 @@ Here's a sample response for a request with an authorization error:
|
||||
< }
|
||||
```
|
||||
|
||||
There are what the HTTP status codes mean:
|
||||
### Responses
|
||||
|
||||
* 200 - The user is able to read the object.
|
||||
* 404 - The repository does not exist for the user, or the user does not have
|
||||
access to it.
|
||||
|
||||
## OPTIONS /objects/{oid}
|
||||
|
||||
This is a pre-flight request to verify credentials before sending the file
|
||||
contents. Note: The `OPTIONS` method is only supported in pre-1.0 Hawser
|
||||
clients. After 1.0, clients should use the `GET` with the
|
||||
`application/vnd.hawser+json` Accept header.
|
||||
|
||||
Here's an example successful request:
|
||||
|
||||
```
|
||||
> OPTIONS https://hawser-server.com/objects/{OID} HTTP/1.1
|
||||
> Accept: application/vnd.hawser+json
|
||||
> Authorization: Basic ... (if authentication is needed)
|
||||
>
|
||||
< HTTP/1.1 200 OK
|
||||
|
||||
(no response body)
|
||||
```
|
||||
|
||||
There are what the HTTP status codes mean:
|
||||
|
||||
* 200 - The user is able to read the object.
|
||||
* 204 - The user is able to PUT the object to the same URL.
|
||||
* 403 - The user has **read**, but not **write** access.
|
||||
* 404 - The repository does not exist for the user.
|
||||
* 405 - OPTIONS not supported, use a GET request with a `application/vnd.hawser+json`
|
||||
Accept header.
|
||||
* 200 - The object exists and the user has access to download it.
|
||||
* 401 - The authentication credentials are incorrect.
|
||||
* 404 - The user does not have access to the object, or it does not exist.
|
||||
|
||||
## POST /objects
|
||||
|
||||
@ -174,44 +187,23 @@ and size of the object to upload.
|
||||
A response can include one of multiple link relations, each with an `href`
|
||||
property and an optional `header` property.
|
||||
|
||||
* `upload` - This relation describes how to upload the object.
|
||||
* `upload` - This relation describes how to upload the object. Expect this with
|
||||
a 202 status.
|
||||
* `verify` - The server can specify a URL for the client to hit after
|
||||
successfully uploading an object.
|
||||
* `download` - This relation describes how to download the object content.
|
||||
successfully uploading an object. This is an optional relation for a 202
|
||||
status.
|
||||
* `download` - This relation describes how to download the object content. This
|
||||
only appears on a 200 status.
|
||||
|
||||
### Responses
|
||||
|
||||
* 200 - The object already exists. Don't bother re-uploading.
|
||||
* 202 - The object is ready to be uploaded.Follow the "upload" and optional
|
||||
* 202 - The object is ready to be uploaded. Follow the "upload" and optional
|
||||
"verify" links.
|
||||
* 401 - The authentication credentials are incorrect.
|
||||
* 403 - The user has **read**, but not **write** access.
|
||||
* 404 - The repository does not exist for the user.
|
||||
|
||||
## PUT /objects/{oid}
|
||||
|
||||
This writes the object contents to the Git Media server.
|
||||
|
||||
```
|
||||
> PUT https://hawser-server.com/objects/{oid} HTTP/1.1
|
||||
> Accept: application/vnd.hawser
|
||||
> Content-Type: application/octet-stream
|
||||
> Authorization: Basic ...
|
||||
> Content-Length: 123
|
||||
>
|
||||
> {binary contents}
|
||||
>
|
||||
< HTTP/1.1 200 OK
|
||||
```
|
||||
|
||||
### Responses
|
||||
|
||||
* 200 - The object already exists.
|
||||
* 201 - The object was uploaded successfully.
|
||||
* 403 - The user has **read**, but not **write** access.
|
||||
* 404 - The repository does not exist for the user.
|
||||
* 405 - PUT method is not allowed. Use an OPTIONS or GET pre-flight request to
|
||||
get the current URL to send a file.
|
||||
|
||||
## Verification
|
||||
|
||||
When Hawser clients issue a POST request to initiate an object upload, the
|
||||
|
@ -38,7 +38,7 @@ func Download(oidPath string) (io.ReadCloser, int64, *WrappedError) {
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", gitMediaType)
|
||||
res, wErr := doRequest(req, creds)
|
||||
res, wErr := doHTTPWithCreds(req, creds)
|
||||
|
||||
if wErr != nil {
|
||||
return nil, 0, wErr
|
||||
@ -190,11 +190,10 @@ func callOptions(filehash string) (int, *WrappedError) {
|
||||
return 0, Errorf(err, "Unable to build OPTIONS request for %s", oid)
|
||||
}
|
||||
|
||||
res, wErr := doRequest(req, creds)
|
||||
res, wErr := doHTTPWithCreds(req, creds)
|
||||
if wErr != nil {
|
||||
return 0, wErr
|
||||
}
|
||||
tracerx.Printf("api_options_status: %d", res.StatusCode)
|
||||
|
||||
return res.StatusCode, nil
|
||||
}
|
||||
@ -240,8 +239,7 @@ func callPut(filehash, filename string, cb CopyCallback) *WrappedError {
|
||||
fmt.Printf("Sending %s\n", filename)
|
||||
|
||||
tracerx.Printf("api_put: %s %s", oid, filename)
|
||||
res, wErr := doRequest(req, creds)
|
||||
tracerx.Printf("api_put_status: %d", res.StatusCode)
|
||||
_, wErr := doHTTPWithCreds(req, creds)
|
||||
|
||||
return wErr
|
||||
}
|
||||
@ -296,7 +294,6 @@ func callExternalPut(filehash, filename string, obj *objectResource, cb CopyCall
|
||||
if err != nil {
|
||||
return Errorf(err, "Error attempting to PUT %s", filename)
|
||||
}
|
||||
tracerx.Printf("external_put_status: %d", res.StatusCode)
|
||||
saveCredentials(creds, res)
|
||||
|
||||
// Run the verify callback
|
||||
@ -323,7 +320,6 @@ func callExternalPut(filehash, filename string, obj *objectResource, cb CopyCall
|
||||
if err != nil {
|
||||
return Errorf(err, "Error attempting to verify %s", filename)
|
||||
}
|
||||
tracerx.Printf("verify_status: %d", verifyRes.StatusCode)
|
||||
saveCredentials(verifyCreds, verifyRes)
|
||||
|
||||
return nil
|
||||
@ -354,11 +350,10 @@ func callPost(filehash, filename string) (*objectResource, int, *WrappedError) {
|
||||
req.Header.Set("Accept", gitMediaMetaType)
|
||||
|
||||
tracerx.Printf("api_post: %s %s", oid, filename)
|
||||
res, wErr := doRequest(req, creds)
|
||||
res, wErr := doHTTPWithCreds(req, creds)
|
||||
if wErr != nil {
|
||||
return nil, 0, wErr
|
||||
}
|
||||
tracerx.Printf("api_post_status: %d", res.StatusCode)
|
||||
|
||||
if res.StatusCode == 202 {
|
||||
obj := &objectResource{}
|
||||
@ -404,7 +399,9 @@ func validateMediaHeader(contentType string, reader io.Reader) (bool, int, *Wrap
|
||||
return true, headerSize, nil
|
||||
}
|
||||
|
||||
func doRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
|
||||
// Wraps DoHTTP(), and saves or removes credentials from the git credential
|
||||
// store based on the response.
|
||||
func doHTTPWithCreds(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
|
||||
res, err := DoHTTP(Config, req)
|
||||
|
||||
var wErr *WrappedError
|
||||
@ -475,7 +472,7 @@ func saveCredentials(creds Creds, res *http.Response) {
|
||||
return
|
||||
}
|
||||
|
||||
if res.StatusCode < 405 {
|
||||
if res.StatusCode < 404 {
|
||||
execCreds(creds, "reject")
|
||||
}
|
||||
}
|
||||
@ -537,10 +534,18 @@ func setRequestHeaders(req *http.Request) (Creds, error) {
|
||||
}
|
||||
|
||||
type ClientError struct {
|
||||
Message string `json:"message"`
|
||||
RequestId string `json:"request_id,omitempty"`
|
||||
Message string `json:"message"`
|
||||
DocumentationUrl string `json:"documentation_url,omitempty"`
|
||||
RequestId string `json:"request_id,omitempty"`
|
||||
}
|
||||
|
||||
func (e *ClientError) Error() string {
|
||||
return e.Message
|
||||
msg := e.Message
|
||||
if len(e.DocumentationUrl) > 0 {
|
||||
msg += "\nDocs: " + e.DocumentationUrl
|
||||
}
|
||||
if len(e.RequestId) > 0 {
|
||||
msg += "\nRequest ID: " + e.RequestId
|
||||
}
|
||||
return msg
|
||||
}
|
||||
|
@ -17,15 +17,24 @@ type Configuration struct {
|
||||
remotes []string
|
||||
httpClient *http.Client
|
||||
redirectingHttpClient *http.Client
|
||||
isTracingHttp bool
|
||||
}
|
||||
|
||||
var (
|
||||
Config = &Configuration{CurrentRemote: defaultRemote}
|
||||
Config = NewConfig()
|
||||
RedirectError = fmt.Errorf("Unexpected redirection")
|
||||
httpPrefixRe = regexp.MustCompile("\\Ahttps?://")
|
||||
defaultRemote = "origin"
|
||||
)
|
||||
|
||||
func NewConfig() *Configuration {
|
||||
c := &Configuration{
|
||||
CurrentRemote: defaultRemote,
|
||||
isTracingHttp: len(os.Getenv("GIT_CURL_VERBOSE")) > 0,
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Configuration) Endpoint() string {
|
||||
if url, ok := c.GitConfig("hawser.url"); ok {
|
||||
return url
|
||||
@ -69,24 +78,18 @@ func (c *Configuration) RemoteEndpoint(remote string) string {
|
||||
}
|
||||
|
||||
func (c *Configuration) Remotes() []string {
|
||||
if c.remotes == nil {
|
||||
c.loadGitConfig()
|
||||
}
|
||||
c.loadGitConfig()
|
||||
return c.remotes
|
||||
}
|
||||
|
||||
func (c *Configuration) GitConfig(key string) (string, bool) {
|
||||
if c.gitConfig == nil {
|
||||
c.loadGitConfig()
|
||||
}
|
||||
c.loadGitConfig()
|
||||
value, ok := c.gitConfig[strings.ToLower(key)]
|
||||
return value, ok
|
||||
}
|
||||
|
||||
func (c *Configuration) SetConfig(key, value string) {
|
||||
if c.gitConfig == nil {
|
||||
c.loadGitConfig()
|
||||
}
|
||||
c.loadGitConfig()
|
||||
c.gitConfig[key] = value
|
||||
}
|
||||
|
||||
@ -107,6 +110,10 @@ type AltConfig struct {
|
||||
}
|
||||
|
||||
func (c *Configuration) loadGitConfig() {
|
||||
if c.gitConfig != nil {
|
||||
return
|
||||
}
|
||||
|
||||
uniqRemotes := make(map[string]bool)
|
||||
|
||||
c.gitConfig = make(map[string]string)
|
||||
@ -148,10 +155,3 @@ func (c *Configuration) loadGitConfig() {
|
||||
c.remotes = append(c.remotes, remote)
|
||||
}
|
||||
}
|
||||
|
||||
func configFileExists(filename string) bool {
|
||||
if _, err := os.Stat(filename); err == nil {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
118
hawser/http.go
118
hawser/http.go
@ -2,17 +2,36 @@ package hawser
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"github.com/rubyist/tracerx"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func DoHTTP(c *Configuration, req *http.Request) (*http.Response, error) {
|
||||
var res *http.Response
|
||||
var err error
|
||||
|
||||
var counter *countingBody
|
||||
if req.Body != nil {
|
||||
counter = newCountingBody(req.Body)
|
||||
req.Body = counter
|
||||
}
|
||||
|
||||
traceHttpRequest(c, req)
|
||||
|
||||
switch req.Method {
|
||||
case "GET", "HEAD":
|
||||
return c.RedirectingHttpClient().Do(req)
|
||||
res, err = c.RedirectingHttpClient().Do(req)
|
||||
default:
|
||||
return c.HttpClient().Do(req)
|
||||
res, err = c.HttpClient().Do(req)
|
||||
}
|
||||
|
||||
traceHttpResponse(c, res, counter)
|
||||
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (c *Configuration) HttpClient() *http.Client {
|
||||
@ -29,18 +48,97 @@ func (c *Configuration) HttpClient() *http.Client {
|
||||
|
||||
func (c *Configuration) RedirectingHttpClient() *http.Client {
|
||||
if c.redirectingHttpClient == nil {
|
||||
c.redirectingHttpClient = &http.Client{
|
||||
Transport: httpTransportFor(c),
|
||||
tr := &http.Transport{}
|
||||
sslVerify, _ := c.GitConfig("http.sslverify")
|
||||
if sslVerify == "false" || len(os.Getenv("GIT_SSL_NO_VERIFY")) > 0 {
|
||||
tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
||||
}
|
||||
c.redirectingHttpClient = &http.Client{Transport: tr}
|
||||
}
|
||||
return c.redirectingHttpClient
|
||||
}
|
||||
|
||||
func httpTransportFor(c *Configuration) *http.Transport {
|
||||
tr := &http.Transport{}
|
||||
sslVerify, _ := c.GitConfig("http.sslverify")
|
||||
if len(os.Getenv("GIT_SSL_NO_VERIFY")) > 0 || sslVerify == "false" {
|
||||
tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
||||
var tracedTypes = []string{"json", "text", "xml", "html"}
|
||||
|
||||
func traceHttpRequest(c *Configuration, req *http.Request) {
|
||||
tracerx.Printf("HTTP: %s %s", req.Method, req.URL.String())
|
||||
|
||||
if c.isTracingHttp == false {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "> %s %s %s\n", req.Method, req.URL.RequestURI(), req.Proto)
|
||||
for key, _ := range req.Header {
|
||||
fmt.Fprintf(os.Stderr, "> %s: %s\n", key, req.Header.Get(key))
|
||||
}
|
||||
return tr
|
||||
}
|
||||
|
||||
func traceHttpResponse(c *Configuration, res *http.Response, counter *countingBody) {
|
||||
tracerx.Printf("HTTP: %d", res.StatusCode)
|
||||
|
||||
if c.isTracingHttp == false {
|
||||
return
|
||||
}
|
||||
|
||||
if counter != nil {
|
||||
fmt.Fprintf(os.Stderr, "* upload sent off: %d bytes\n", counter.Size)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "\n")
|
||||
|
||||
fmt.Fprintf(os.Stderr, "< %s %s\n", res.Proto, res.Status)
|
||||
for key, _ := range res.Header {
|
||||
fmt.Fprintf(os.Stderr, "< %s: %s\n", key, res.Header.Get(key))
|
||||
}
|
||||
|
||||
traceBody := false
|
||||
ctype := strings.ToLower(strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0])
|
||||
for _, tracedType := range tracedTypes {
|
||||
if strings.Contains(ctype, tracedType) {
|
||||
traceBody = true
|
||||
}
|
||||
}
|
||||
|
||||
if traceBody {
|
||||
fmt.Fprintf(os.Stderr, "\n")
|
||||
res.Body = newTracedBody(res.Body)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\n")
|
||||
}
|
||||
|
||||
type countingBody struct {
|
||||
body io.ReadCloser
|
||||
Size int64
|
||||
}
|
||||
|
||||
func (r *countingBody) Read(p []byte) (int, error) {
|
||||
n, err := r.body.Read(p)
|
||||
r.Size += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *countingBody) Close() error {
|
||||
return r.body.Close()
|
||||
}
|
||||
|
||||
func newCountingBody(body io.ReadCloser) *countingBody {
|
||||
return &countingBody{body, 0}
|
||||
}
|
||||
|
||||
type tracedBody struct {
|
||||
body io.ReadCloser
|
||||
}
|
||||
|
||||
func (r *tracedBody) Read(p []byte) (int, error) {
|
||||
n, err := r.body.Read(p)
|
||||
fmt.Fprintf(os.Stderr, "%s\n", string(p[0:n]))
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *tracedBody) Close() error {
|
||||
return r.body.Close()
|
||||
}
|
||||
|
||||
func newTracedBody(body io.ReadCloser) *tracedBody {
|
||||
return &tracedBody{body}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user