// +build testtools package main import ( "bufio" "bytes" "compress/gzip" "crypto/rand" "crypto/rsa" "crypto/sha256" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/base64" "encoding/hex" "encoding/json" "encoding/pem" "errors" "fmt" "io" "io/ioutil" "log" "math" "math/big" "net/http" "net/http/httptest" "net/textproto" "os" "os/exec" "regexp" "sort" "strconv" "strings" "sync" "time" ) var ( repoDir string largeObjects = newLfsStorage() server *httptest.Server serverTLS *httptest.Server serverClientCert *httptest.Server // maps OIDs to content strings. Both the LFS and Storage test servers below // see OIDs. oidHandlers map[string]string // These magic strings tell the test lfs server change their behavior so the // integration tests can check those use cases. Tests will create objects with // the magic strings as the contents. // // printf "status:lfs:404" > 404.dat // contentHandlers = []string{ "status-batch-403", "status-batch-404", "status-batch-410", "status-batch-422", "status-batch-500", "status-storage-403", "status-storage-404", "status-storage-410", "status-storage-422", "status-storage-500", "status-storage-503", "status-batch-resume-206", "batch-resume-fail-fallback", "return-expired-action", "return-expired-action-forever", "return-invalid-size", "object-authenticated", "storage-download-retry", "storage-upload-retry", "storage-upload-retry-later", "unknown-oid", "send-verify-action", "send-deprecated-links", "redirect-storage-upload", "storage-compress", } reqCookieReposRE = regexp.MustCompile(`\A/require-cookie-`) ) func main() { repoDir = os.Getenv("LFSTEST_DIR") mux := http.NewServeMux() server = httptest.NewServer(mux) serverTLS = httptest.NewTLSServer(mux) serverClientCert = httptest.NewUnstartedServer(mux) //setup Client Cert server rootKey, rootCert := generateCARootCertificates() _, clientCertPEM, clientKeyPEM, clientKeyEncPEM := generateClientCertificates(rootCert, rootKey) certPool := x509.NewCertPool() certPool.AddCert(rootCert) serverClientCert.TLS = &tls.Config{ Certificates: []tls.Certificate{serverTLS.TLS.Certificates[0]}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: certPool, } serverClientCert.StartTLS() stopch := make(chan bool) mux.HandleFunc("/shutdown", func(w http.ResponseWriter, r *http.Request) { stopch <- true }) mux.HandleFunc("/storage/", storageHandler) mux.HandleFunc("/verify", verifyHandler) mux.HandleFunc("/redirect307/", redirect307Handler) mux.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "%s\n", time.Now().String()) }) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { id, ok := reqId(w) if !ok { return } if reqCookieReposRE.MatchString(r.URL.Path) { if skipIfNoCookie(w, r, id) { return } } if strings.Contains(r.URL.Path, "/info/lfs") { if !skipIfBadAuth(w, r, id) { lfsHandler(w, r, id) } return } debug(id, "git http-backend %s %s", r.Method, r.URL) gitHandler(w, r) }) urlname := writeTestStateFile([]byte(server.URL), "LFSTEST_URL", "lfstest-gitserver") defer os.RemoveAll(urlname) sslurlname := writeTestStateFile([]byte(serverTLS.URL), "LFSTEST_SSL_URL", "lfstest-gitserver-ssl") defer os.RemoveAll(sslurlname) clientCertUrlname := writeTestStateFile([]byte(serverClientCert.URL), "LFSTEST_CLIENT_CERT_URL", "lfstest-gitserver-ssl") defer os.RemoveAll(clientCertUrlname) block := &pem.Block{} block.Type = "CERTIFICATE" block.Bytes = serverTLS.TLS.Certificates[0].Certificate[0] pembytes := pem.EncodeToMemory(block) certname := writeTestStateFile(pembytes, "LFSTEST_CERT", "lfstest-gitserver-cert") defer os.RemoveAll(certname) cccertname := writeTestStateFile(clientCertPEM, "LFSTEST_CLIENT_CERT", "lfstest-gitserver-client-cert") defer os.RemoveAll(cccertname) ckcertname := writeTestStateFile(clientKeyPEM, "LFSTEST_CLIENT_KEY", "lfstest-gitserver-client-key") defer os.RemoveAll(ckcertname) ckecertname := writeTestStateFile(clientKeyEncPEM, "LFSTEST_CLIENT_KEY_ENCRYPTED", "lfstest-gitserver-client-key-enc") defer os.RemoveAll(ckecertname) debug("init", "server url: %s", server.URL) debug("init", "server tls url: %s", serverTLS.URL) debug("init", "server client cert url: %s", serverClientCert.URL) <-stopch debug("init", "git server done") } // writeTestStateFile writes contents to either the file referenced by the // environment variable envVar, or defaultFilename if that's not set. Returns // the filename that was used func writeTestStateFile(contents []byte, envVar, defaultFilename string) string { f := os.Getenv(envVar) if len(f) == 0 { f = defaultFilename } file, err := os.Create(f) if err != nil { log.Fatalln(err) } file.Write(contents) file.Close() return f } type lfsObject struct { Oid string `json:"oid,omitempty"` Size int64 `json:"size,omitempty"` Authenticated bool `json:"authenticated,omitempty"` Actions map[string]*lfsLink `json:"actions,omitempty"` Links map[string]*lfsLink `json:"_links,omitempty"` Err *lfsError `json:"error,omitempty"` } type lfsLink struct { Href string `json:"href"` Header map[string]string `json:"header,omitempty"` ExpiresAt time.Time `json:"expires_at,omitempty"` ExpiresIn int `json:"expires_in,omitempty"` } type lfsError struct { Code int `json:"code,omitempty"` Message string `json:"message"` } func writeLFSError(w http.ResponseWriter, code int, msg string) { by, err := json.Marshal(&lfsError{Message: msg}) if err != nil { http.Error(w, "json encoding error: "+err.Error(), 500) return } w.Header().Set("Content-Type", "application/vnd.git-lfs+json") w.WriteHeader(code) w.Write(by) } // handles any requests with "{name}.server.git/info/lfs" in the path func lfsHandler(w http.ResponseWriter, r *http.Request, id string) { repo, err := repoFromLfsUrl(r.URL.Path) if err != nil { w.WriteHeader(500) w.Write([]byte(err.Error())) return } // Check that we're sending valid data. if !strings.Contains(r.Header.Get("Accept"), "application/vnd.git-lfs+json") { w.WriteHeader(406) return } debug(id, "git lfs %s %s repo: %s", r.Method, r.URL, repo) w.Header().Set("Content-Type", "application/vnd.git-lfs+json") switch r.Method { case "POST": // Reject invalid data. if !strings.Contains(r.Header.Get("Content-Type"), "application/vnd.git-lfs+json") { w.WriteHeader(400) return } if strings.HasSuffix(r.URL.String(), "batch") { lfsBatchHandler(w, r, id, repo) } else { locksHandler(w, r, repo) } case "DELETE": lfsDeleteHandler(w, r, id, repo) case "GET": if strings.Contains(r.URL.String(), "/locks") { locksHandler(w, r, repo) } else { w.WriteHeader(404) w.Write([]byte("lock request")) } default: w.WriteHeader(405) } } func lfsUrl(repo, oid string, redirect bool) string { if redirect { return server.URL + "/redirect307/objects/" + oid + "?r=" + repo } return server.URL + "/storage/" + oid + "?r=" + repo } const ( secondsToRefillTokens = 10 refillTokenCount = 5 ) var ( requestTokens = make(map[string]int) retryStartTimes = make(map[string]time.Time) laterRetriesMu sync.Mutex ) // checkRateLimit tracks the various requests to the git-server. If it is the first // request of its kind, then a times is started, that when it is finished, a certain // number of requests become available. func checkRateLimit(api, direction, repo, oid string) (seconds int, isWait bool) { laterRetriesMu.Lock() defer laterRetriesMu.Unlock() key := strings.Join([]string{direction, repo, oid}, ":") if requestsRemaining, ok := requestTokens[key]; !ok || requestsRemaining == 0 { if retryStartTimes[key] == (time.Time{}) { // If time is not initialized, set it to now retryStartTimes[key] = time.Now() } // The user is not allowed to make a request now and must wait for the required // time to pass. secsPassed := time.Since(retryStartTimes[key]).Seconds() if secsPassed >= float64(secondsToRefillTokens) { // The required time has passed. requestTokens[key] = refillTokenCount return 0, false } return secondsToRefillTokens - int(secsPassed) + 1, true } requestTokens[key]-- // Tokens are now over, record time. if requestTokens[key] == 0 { retryStartTimes[key] = time.Now() } return 0, false } var ( retries = make(map[string]uint32) retriesMu sync.Mutex ) func incrementRetriesFor(api, direction, repo, oid string, check bool) (after uint32, ok bool) { // fmtStr formats a string like "--[check]-", // i.e., "legacy-upload-check-retry", or "storage-download-retry". var fmtStr string if check { fmtStr = "%s-%s-check-retry" } else { fmtStr = "%s-%s-retry" } if oidHandlers[oid] != fmt.Sprintf(fmtStr, api, direction) { return 0, false } retriesMu.Lock() defer retriesMu.Unlock() retryKey := strings.Join([]string{direction, repo, oid}, ":") retries[retryKey]++ retries := retries[retryKey] return retries, true } func lfsDeleteHandler(w http.ResponseWriter, r *http.Request, id, repo string) { parts := strings.Split(r.URL.Path, "/") oid := parts[len(parts)-1] largeObjects.Delete(repo, oid) debug(id, "DELETE:", oid) w.WriteHeader(200) } type batchReq struct { Transfers []string `json:"transfers"` Operation string `json:"operation"` Objects []lfsObject `json:"objects"` Ref *Ref `json:"ref,omitempty"` } func (r *batchReq) RefName() string { if r.Ref == nil { return "" } return r.Ref.Name } type batchResp struct { Transfer string `json:"transfer,omitempty"` Objects []lfsObject `json:"objects"` } func lfsBatchHandler(w http.ResponseWriter, r *http.Request, id, repo string) { checkingObject := r.Header.Get("X-Check-Object") == "1" if !checkingObject && repo == "batchunsupported" { w.WriteHeader(404) return } if !checkingObject && repo == "badbatch" { w.WriteHeader(203) return } if repo == "netrctest" { user, pass, err := extractAuth(r.Header.Get("Authorization")) if err != nil || (user != "netrcuser" || pass != "netrcpass") { w.WriteHeader(403) return } } if missingRequiredCreds(w, r, repo) { return } buf := &bytes.Buffer{} tee := io.TeeReader(r.Body, buf) objs := &batchReq{} err := json.NewDecoder(tee).Decode(objs) io.Copy(ioutil.Discard, r.Body) r.Body.Close() debug(id, "REQUEST") debug(id, buf.String()) if err != nil { log.Fatal(err) } if strings.HasSuffix(repo, "branch-required") { parts := strings.Split(repo, "-") lenParts := len(parts) if lenParts > 3 && "refs/heads/"+parts[lenParts-3] != objs.RefName() { w.WriteHeader(403) json.NewEncoder(w).Encode(struct { Message string `json:"message"` }{fmt.Sprintf("Expected ref %q, got %q", "refs/heads/"+parts[lenParts-3], objs.RefName())}) return } } res := []lfsObject{} testingChunked := testingChunkedTransferEncoding(r) testingTus := testingTusUploadInBatchReq(r) testingTusInterrupt := testingTusUploadInterruptedInBatchReq(r) testingCustomTransfer := testingCustomTransfer(r) var transferChoice string var searchForTransfer string if testingTus { searchForTransfer = "tus" } else if testingCustomTransfer { searchForTransfer = "testcustom" } if len(searchForTransfer) > 0 { for _, t := range objs.Transfers { if t == searchForTransfer { transferChoice = searchForTransfer break } } } for _, obj := range objs.Objects { handler := oidHandlers[obj.Oid] action := objs.Operation o := lfsObject{ Size: obj.Size, Actions: make(map[string]*lfsLink), } // Clobber the OID if told to do so. if handler == "unknown-oid" { o.Oid = "unknown-oid" } else { o.Oid = obj.Oid } exists := largeObjects.Has(repo, obj.Oid) addAction := true if action == "download" { if !exists { o.Err = &lfsError{Code: 404, Message: fmt.Sprintf("Object %v does not exist", obj.Oid)} addAction = false } } else { if exists { // not an error but don't add an action addAction = false } } if handler == "object-authenticated" { o.Authenticated = true } switch handler { case "status-batch-403": o.Err = &lfsError{Code: 403, Message: "welp"} case "status-batch-404": o.Err = &lfsError{Code: 404, Message: "welp"} case "status-batch-410": o.Err = &lfsError{Code: 410, Message: "welp"} case "status-batch-422": o.Err = &lfsError{Code: 422, Message: "welp"} case "status-batch-500": o.Err = &lfsError{Code: 500, Message: "welp"} default: // regular 200 response if handler == "return-invalid-size" { o.Size = -1 } if handler == "send-deprecated-links" { o.Links = make(map[string]*lfsLink) } if addAction { a := &lfsLink{ Href: lfsUrl(repo, obj.Oid, handler == "redirect-storage-upload"), Header: map[string]string{}, } a = serveExpired(a, repo, handler) if handler == "send-deprecated-links" { o.Links[action] = a } else { o.Actions[action] = a } } if handler == "send-verify-action" { o.Actions["verify"] = &lfsLink{ Href: server.URL + "/verify", Header: map[string]string{ "repo": repo, }, } } } if testingChunked && addAction { if handler == "send-deprecated-links" { o.Links[action].Header["Transfer-Encoding"] = "chunked" } else { o.Actions[action].Header["Transfer-Encoding"] = "chunked" } } if testingTusInterrupt && addAction { if handler == "send-deprecated-links" { o.Links[action].Header["Lfs-Tus-Interrupt"] = "true" } else { o.Actions[action].Header["Lfs-Tus-Interrupt"] = "true" } } res = append(res, o) } ores := batchResp{Transfer: transferChoice, Objects: res} by, err := json.Marshal(ores) if err != nil { log.Fatal(err) } debug(id, "RESPONSE: 200") debug(id, string(by)) w.WriteHeader(200) w.Write(by) } // emu guards expiredRepos var emu sync.Mutex // expiredRepos is a map keyed by repository name, valuing to whether or not it // has yet served an expired object. var expiredRepos = map[string]bool{} // serveExpired marks the given repo as having served an expired object, making // it unable for that same repository to return an expired object in the future, func serveExpired(a *lfsLink, repo, handler string) *lfsLink { var ( dur = -5 * time.Minute at = time.Now().Add(dur) ) if handler == "return-expired-action-forever" || (handler == "return-expired-action" && canServeExpired(repo)) { emu.Lock() expiredRepos[repo] = true emu.Unlock() a.ExpiresAt = at return a } switch repo { case "expired-absolute": a.ExpiresAt = at case "expired-relative": a.ExpiresIn = -5 case "expired-both": a.ExpiresAt = at a.ExpiresIn = -5 } return a } // canServeExpired returns whether or not a repository is capable of serving an // expired object. In other words, canServeExpired returns whether or not the // given repo has yet served an expired object. func canServeExpired(repo string) bool { emu.Lock() defer emu.Unlock() return !expiredRepos[repo] } // Persistent state across requests var batchResumeFailFallbackStorageAttempts = 0 var tusStorageAttempts = 0 var ( vmu sync.Mutex verifyCounts = make(map[string]int) verifyRetryRe = regexp.MustCompile(`verify-fail-(\d+)-times?$`) ) func verifyHandler(w http.ResponseWriter, r *http.Request) { repo := r.Header.Get("repo") var payload struct { Oid string `json:"oid"` Size int64 `json:"size"` } if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { writeLFSError(w, http.StatusUnprocessableEntity, err.Error()) return } var max int if matches := verifyRetryRe.FindStringSubmatch(repo); len(matches) < 2 { return } else { max, _ = strconv.Atoi(matches[1]) } key := strings.Join([]string{repo, payload.Oid}, ":") vmu.Lock() verifyCounts[key] = verifyCounts[key] + 1 count := verifyCounts[key] vmu.Unlock() if count < max { writeLFSError(w, http.StatusServiceUnavailable, fmt.Sprintf( "intentionally failing verify request %d (out of %d)", count, max, )) return } } // handles any /storage/{oid} requests func storageHandler(w http.ResponseWriter, r *http.Request) { id, ok := reqId(w) if !ok { return } repo := r.URL.Query().Get("r") parts := strings.Split(r.URL.Path, "/") oid := parts[len(parts)-1] if missingRequiredCreds(w, r, repo) { return } debug(id, "storage %s %s repo: %s", r.Method, oid, repo) switch r.Method { case "PUT": switch oidHandlers[oid] { case "status-storage-403": w.WriteHeader(403) return case "status-storage-404": w.WriteHeader(404) return case "status-storage-410": w.WriteHeader(410) return case "status-storage-422": w.WriteHeader(422) return case "status-storage-500": w.WriteHeader(500) return case "status-storage-503": writeLFSError(w, 503, "LFS is temporarily unavailable") return case "object-authenticated": if len(r.Header.Get("Authorization")) > 0 { w.WriteHeader(400) w.Write([]byte("Should not send authentication")) } return case "storage-upload-retry": if retries, ok := incrementRetriesFor("storage", "upload", repo, oid, false); ok && retries < 3 { w.WriteHeader(500) w.Write([]byte("malformed content")) return } case "storage-upload-retry-later": if timeLeft, isWaiting := checkRateLimit("storage", "upload", repo, oid); isWaiting { w.Header().Set("Retry-After", strconv.Itoa(timeLeft)) w.WriteHeader(http.StatusTooManyRequests) w.Write([]byte("rate limit reached")) fmt.Println("Setting header to: ", strconv.Itoa(timeLeft)) return } case "storage-compress": if r.Header.Get("Accept-Encoding") != "gzip" { w.WriteHeader(500) w.Write([]byte("not encoded")) return } } if testingChunkedTransferEncoding(r) { valid := false for _, value := range r.TransferEncoding { if value == "chunked" { valid = true break } } if !valid { debug(id, "Chunked transfer encoding expected") } } hash := sha256.New() buf := &bytes.Buffer{} io.Copy(io.MultiWriter(hash, buf), r.Body) oid := hex.EncodeToString(hash.Sum(nil)) if !strings.HasSuffix(r.URL.Path, "/"+oid) { w.WriteHeader(403) return } largeObjects.Set(repo, oid, buf.Bytes()) case "GET": parts := strings.Split(r.URL.Path, "/") oid := parts[len(parts)-1] statusCode := 200 byteLimit := 0 resumeAt := int64(0) compress := false if by, ok := largeObjects.Get(repo, oid); ok { if len(by) == len("storage-download-retry-later") && string(by) == "storage-download-retry-later" { if secsToWait, wait := checkRateLimit("storage", "download", repo, oid); wait { statusCode = http.StatusTooManyRequests w.Header().Set("Retry-After", strconv.Itoa(secsToWait)) by = []byte("rate limit reached") fmt.Println("Setting header to: ", strconv.Itoa(secsToWait)) } } else if len(by) == len("storage-download-retry") && string(by) == "storage-download-retry" { if retries, ok := incrementRetriesFor("storage", "download", repo, oid, false); ok && retries < 3 { statusCode = 500 by = []byte("malformed content") } } else if len(by) == len("storage-compress") && string(by) == "storage-compress" { if r.Header.Get("Accept-Encoding") != "gzip" { statusCode = 500 by = []byte("not encoded") } else { compress = true } } else if len(by) == len("status-batch-resume-206") && string(by) == "status-batch-resume-206" { // Resume if header includes range, otherwise deliberately interrupt if rangeHdr := r.Header.Get("Range"); rangeHdr != "" { regex := regexp.MustCompile(`bytes=(\d+)\-.*`) match := regex.FindStringSubmatch(rangeHdr) if match != nil && len(match) > 1 { statusCode = 206 resumeAt, _ = strconv.ParseInt(match[1], 10, 32) w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", resumeAt, len(by), resumeAt-int64(len(by)))) } } else { byteLimit = 10 } } else if len(by) == len("batch-resume-fail-fallback") && string(by) == "batch-resume-fail-fallback" { // Fail any Range: request even though we said we supported it // To make sure client can fall back if rangeHdr := r.Header.Get("Range"); rangeHdr != "" { w.WriteHeader(416) return } if batchResumeFailFallbackStorageAttempts == 0 { // Truncate output on FIRST attempt to cause resume // Second attempt (without range header) is fallback, complete successfully byteLimit = 8 batchResumeFailFallbackStorageAttempts++ } } else if string(by) == "status-batch-retry" { if rangeHdr := r.Header.Get("Range"); rangeHdr != "" { regex := regexp.MustCompile(`bytes=(\d+)\-(.*)`) match := regex.FindStringSubmatch(rangeHdr) // We have a Range header with two // non-empty values. if match != nil && len(match) > 2 && len(match[2]) != 0 { first, _ := strconv.ParseInt(match[1], 10, 32) second, _ := strconv.ParseInt(match[2], 10, 32) // The second part of the range // is smaller than the first // part (or the latter part of // the range is non-integral). // This is invalid; reject it. if second < first { w.WriteHeader(400) return } // The range is valid; we'll // take the branch below. } // We got a valid range header, so // provide a 206 Partial Content. We // ignore the upper bound if one was // provided. if match != nil && len(match) > 1 { statusCode = 206 resumeAt, _ = strconv.ParseInt(match[1], 10, 32) w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", resumeAt, len(by), resumeAt-int64(len(by)))) } } } var wrtr io.Writer = w if compress { w.Header().Set("Content-Encoding", "gzip") gz := gzip.NewWriter(w) defer gz.Close() wrtr = gz } w.WriteHeader(statusCode) if byteLimit > 0 { wrtr.Write(by[0:byteLimit]) } else if resumeAt > 0 { wrtr.Write(by[resumeAt:]) } else { wrtr.Write(by) } return } w.WriteHeader(404) case "HEAD": // tus.io if !validateTusHeaders(r, id) { w.WriteHeader(400) return } parts := strings.Split(r.URL.Path, "/") oid := parts[len(parts)-1] var offset int64 if by, ok := largeObjects.GetIncomplete(repo, oid); ok { offset = int64(len(by)) } w.Header().Set("Upload-Offset", strconv.FormatInt(offset, 10)) w.WriteHeader(200) case "PATCH": // tus.io if !validateTusHeaders(r, id) { w.WriteHeader(400) return } parts := strings.Split(r.URL.Path, "/") oid := parts[len(parts)-1] offsetHdr := r.Header.Get("Upload-Offset") offset, err := strconv.ParseInt(offsetHdr, 10, 64) if err != nil { log.Fatal("Unable to parse Upload-Offset header in request: ", err) w.WriteHeader(400) return } hash := sha256.New() buf := &bytes.Buffer{} out := io.MultiWriter(hash, buf) if by, ok := largeObjects.GetIncomplete(repo, oid); ok { if offset != int64(len(by)) { log.Fatal(fmt.Sprintf("Incorrect offset in request, got %d expected %d", offset, len(by))) w.WriteHeader(400) return } _, err := out.Write(by) if err != nil { log.Fatal("Error reading incomplete bytes from store: ", err) w.WriteHeader(500) return } largeObjects.DeleteIncomplete(repo, oid) debug(id, "Resuming upload of %v at byte %d", oid, offset) } // As a test, we intentionally break the upload from byte 0 by only // reading some bytes the quitting & erroring, this forces a resume // any offset > 0 will work ok var copyErr error if r.Header.Get("Lfs-Tus-Interrupt") == "true" && offset == 0 { chdr := r.Header.Get("Content-Length") contentLen, err := strconv.ParseInt(chdr, 10, 64) if err != nil { log.Fatal(fmt.Sprintf("Invalid Content-Length %q", chdr)) w.WriteHeader(400) return } truncated := contentLen / 3 _, _ = io.CopyN(out, r.Body, truncated) r.Body.Close() copyErr = fmt.Errorf("Simulated copy error") } else { _, copyErr = io.Copy(out, r.Body) } if copyErr != nil { b := buf.Bytes() if len(b) > 0 { debug(id, "Incomplete upload of %v, %d bytes", oid, len(b)) largeObjects.SetIncomplete(repo, oid, b) } w.WriteHeader(500) } else { checkoid := hex.EncodeToString(hash.Sum(nil)) if checkoid != oid { log.Fatal(fmt.Sprintf("Incorrect oid after calculation, got %q expected %q", checkoid, oid)) w.WriteHeader(403) return } b := buf.Bytes() largeObjects.Set(repo, oid, b) w.Header().Set("Upload-Offset", strconv.FormatInt(int64(len(b)), 10)) w.WriteHeader(204) } default: w.WriteHeader(405) } } func validateTusHeaders(r *http.Request, id string) bool { if len(r.Header.Get("Tus-Resumable")) == 0 { debug(id, "Missing Tus-Resumable header in request") return false } return true } func gitHandler(w http.ResponseWriter, r *http.Request) { defer func() { io.Copy(ioutil.Discard, r.Body) r.Body.Close() }() cmd := exec.Command("git", "http-backend") cmd.Env = []string{ fmt.Sprintf("GIT_PROJECT_ROOT=%s", repoDir), fmt.Sprintf("GIT_HTTP_EXPORT_ALL="), fmt.Sprintf("PATH_INFO=%s", r.URL.Path), fmt.Sprintf("QUERY_STRING=%s", r.URL.RawQuery), fmt.Sprintf("REQUEST_METHOD=%s", r.Method), fmt.Sprintf("CONTENT_TYPE=%s", r.Header.Get("Content-Type")), } buffer := &bytes.Buffer{} cmd.Stdin = r.Body cmd.Stdout = buffer cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { log.Fatal(err) } text := textproto.NewReader(bufio.NewReader(buffer)) code, _, _ := text.ReadCodeLine(-1) if code != 0 { w.WriteHeader(code) } headers, _ := text.ReadMIMEHeader() head := w.Header() for key, values := range headers { for _, value := range values { head.Add(key, value) } } io.Copy(w, text.R) } func redirect307Handler(w http.ResponseWriter, r *http.Request) { id, ok := reqId(w) if !ok { return } // Send a redirect to info/lfs // Make it either absolute or relative depending on subpath parts := strings.Split(r.URL.Path, "/") // first element is always blank since rooted var redirectTo string if parts[2] == "rel" { redirectTo = "/" + strings.Join(parts[3:], "/") } else if parts[2] == "abs" { redirectTo = server.URL + "/" + strings.Join(parts[3:], "/") } else if parts[2] == "objects" { repo := r.URL.Query().Get("r") redirectTo = server.URL + "/storage/" + strings.Join(parts[3:], "/") + "?r=" + repo } else { debug(id, "Invalid URL for redirect: %v", r.URL) w.WriteHeader(404) return } w.Header().Set("Location", redirectTo) w.WriteHeader(307) } type User struct { Name string `json:"name"` } type Lock struct { Id string `json:"id"` Path string `json:"path"` Owner User `json:"owner"` LockedAt time.Time `json:"locked_at"` } type LockRequest struct { Path string `json:"path"` Ref *Ref `json:"ref,omitempty"` } func (r *LockRequest) RefName() string { if r.Ref == nil { return "" } return r.Ref.Name } type LockResponse struct { Lock *Lock `json:"lock"` Message string `json:"message,omitempty"` } type UnlockRequest struct { Force bool `json:"force"` Ref *Ref `json:"ref,omitempty"` } func (r *UnlockRequest) RefName() string { if r.Ref == nil { return "" } return r.Ref.Name } type UnlockResponse struct { Lock *Lock `json:"lock"` Message string `json:"message,omitempty"` } type LockList struct { Locks []Lock `json:"locks"` NextCursor string `json:"next_cursor,omitempty"` Message string `json:"message,omitempty"` } type Ref struct { Name string `json:"name,omitempty"` } type VerifiableLockRequest struct { Ref *Ref `json:"ref,omitempty"` Cursor string `json:"cursor,omitempty"` Limit int `json:"limit,omitempty"` } func (r *VerifiableLockRequest) RefName() string { if r.Ref == nil { return "" } return r.Ref.Name } type VerifiableLockList struct { Ours []Lock `json:"ours"` Theirs []Lock `json:"theirs"` NextCursor string `json:"next_cursor,omitempty"` Message string `json:"message,omitempty"` } var ( lmu sync.RWMutex repoLocks = map[string][]Lock{} ) func addLocks(repo string, l ...Lock) { lmu.Lock() defer lmu.Unlock() repoLocks[repo] = append(repoLocks[repo], l...) sort.Sort(LocksByCreatedAt(repoLocks[repo])) } func getLocks(repo string) []Lock { lmu.RLock() defer lmu.RUnlock() locks := repoLocks[repo] cp := make([]Lock, len(locks)) for i, l := range locks { cp[i] = l } return cp } func getFilteredLocks(repo, path, cursor, limit string) ([]Lock, string, error) { locks := getLocks(repo) if cursor != "" { lastSeen := -1 for i, l := range locks { if l.Id == cursor { lastSeen = i break } } if lastSeen > -1 { locks = locks[lastSeen:] } else { return nil, "", fmt.Errorf("cursor (%s) not found", cursor) } } if path != "" { var filtered []Lock for _, l := range locks { if l.Path == path { filtered = append(filtered, l) } } locks = filtered } if limit != "" { size, err := strconv.Atoi(limit) if err != nil { return nil, "", errors.New("unable to parse limit amount") } size = int(math.Min(float64(len(locks)), 3)) if size < 0 { return nil, "", nil } if size+1 < len(locks) { return locks[:size], locks[size+1].Id, nil } } return locks, "", nil } func delLock(repo string, id string) *Lock { lmu.RLock() defer lmu.RUnlock() var deleted *Lock locks := make([]Lock, 0, len(repoLocks[repo])) for _, l := range repoLocks[repo] { if l.Id == id { deleted = &l continue } locks = append(locks, l) } repoLocks[repo] = locks return deleted } type LocksByCreatedAt []Lock func (c LocksByCreatedAt) Len() int { return len(c) } func (c LocksByCreatedAt) Less(i, j int) bool { return c[i].LockedAt.Before(c[j].LockedAt) } func (c LocksByCreatedAt) Swap(i, j int) { c[i], c[j] = c[j], c[i] } var ( lockRe = regexp.MustCompile(`/locks/?$`) unlockRe = regexp.MustCompile(`locks/([^/]+)/unlock\z`) ) func locksHandler(w http.ResponseWriter, r *http.Request, repo string) { dec := json.NewDecoder(r.Body) enc := json.NewEncoder(w) if repo == "netrctest" { user, pass, err := extractAuth(r.Header.Get("Authorization")) if err != nil || (user == "netrcuser" && pass == "badpassretry") { writeLFSError(w, 401, "Error: Bad Auth") return } } switch r.Method { case "GET": if !lockRe.MatchString(r.URL.Path) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotFound) w.Write([]byte(`{"message":"unknown path: ` + r.URL.Path + `"}`)) return } if err := r.ParseForm(); err != nil { http.Error(w, "could not parse form values", http.StatusInternalServerError) return } if strings.HasSuffix(repo, "branch-required") { parts := strings.Split(repo, "-") lenParts := len(parts) if lenParts > 3 && "refs/heads/"+parts[lenParts-3] != r.FormValue("refspec") { w.WriteHeader(403) enc.Encode(struct { Message string `json:"message"` }{fmt.Sprintf("Expected ref %q, got %q", "refs/heads/"+parts[lenParts-3], r.FormValue("refspec"))}) return } } ll := &LockList{} w.Header().Set("Content-Type", "application/json") locks, nextCursor, err := getFilteredLocks(repo, r.FormValue("path"), r.FormValue("cursor"), r.FormValue("limit")) if err != nil { ll.Message = err.Error() } else { ll.Locks = locks ll.NextCursor = nextCursor } enc.Encode(ll) return case "POST": w.Header().Set("Content-Type", "application/json") if strings.HasSuffix(r.URL.Path, "unlock") { var lockId string if matches := unlockRe.FindStringSubmatch(r.URL.Path); len(matches) > 1 { lockId = matches[1] } if len(lockId) == 0 { enc.Encode(&UnlockResponse{Message: "Invalid lock"}) } unlockRequest := &UnlockRequest{} if err := dec.Decode(unlockRequest); err != nil { enc.Encode(&UnlockResponse{Message: err.Error()}) return } if strings.HasSuffix(repo, "branch-required") { parts := strings.Split(repo, "-") lenParts := len(parts) if lenParts > 3 && "refs/heads/"+parts[lenParts-3] != unlockRequest.RefName() { w.WriteHeader(403) enc.Encode(struct { Message string `json:"message"` }{fmt.Sprintf("Expected ref %q, got %q", "refs/heads/"+parts[lenParts-3], unlockRequest.RefName())}) return } } if l := delLock(repo, lockId); l != nil { enc.Encode(&UnlockResponse{Lock: l}) } else { enc.Encode(&UnlockResponse{Message: "unable to find lock"}) } return } if strings.HasSuffix(r.URL.Path, "/locks/verify") { if strings.HasSuffix(repo, "verify-5xx") { w.WriteHeader(500) return } if strings.HasSuffix(repo, "verify-501") { w.WriteHeader(501) return } if strings.HasSuffix(repo, "verify-403") { w.WriteHeader(403) return } switch repo { case "pre_push_locks_verify_404": w.WriteHeader(http.StatusNotFound) w.Write([]byte(`{"message":"pre_push_locks_verify_404"}`)) return case "pre_push_locks_verify_410": w.WriteHeader(http.StatusGone) w.Write([]byte(`{"message":"pre_push_locks_verify_410"}`)) return } reqBody := &VerifiableLockRequest{} if err := dec.Decode(reqBody); err != nil { w.WriteHeader(http.StatusBadRequest) enc.Encode(struct { Message string `json:"message"` }{"json decode error: " + err.Error()}) return } if strings.HasSuffix(repo, "branch-required") { parts := strings.Split(repo, "-") lenParts := len(parts) if lenParts > 3 && "refs/heads/"+parts[lenParts-3] != reqBody.RefName() { w.WriteHeader(403) enc.Encode(struct { Message string `json:"message"` }{fmt.Sprintf("Expected ref %q, got %q", "refs/heads/"+parts[lenParts-3], reqBody.RefName())}) return } } ll := &VerifiableLockList{} locks, nextCursor, err := getFilteredLocks(repo, "", reqBody.Cursor, strconv.Itoa(reqBody.Limit)) if err != nil { ll.Message = err.Error() } else { ll.NextCursor = nextCursor for _, l := range locks { if strings.Contains(l.Path, "theirs") { ll.Theirs = append(ll.Theirs, l) } else { ll.Ours = append(ll.Ours, l) } } } enc.Encode(ll) return } if strings.HasSuffix(r.URL.Path, "/locks") { lockRequest := &LockRequest{} if err := dec.Decode(lockRequest); err != nil { enc.Encode(&LockResponse{Message: err.Error()}) } if strings.HasSuffix(repo, "branch-required") { parts := strings.Split(repo, "-") lenParts := len(parts) if lenParts > 3 && "refs/heads/"+parts[lenParts-3] != lockRequest.RefName() { w.WriteHeader(403) enc.Encode(struct { Message string `json:"message"` }{fmt.Sprintf("Expected ref %q, got %q", "refs/heads/"+parts[lenParts-3], lockRequest.RefName())}) return } } for _, l := range getLocks(repo) { if l.Path == lockRequest.Path { enc.Encode(&LockResponse{Message: "lock already created"}) return } } var id [20]byte rand.Read(id[:]) lock := &Lock{ Id: fmt.Sprintf("%x", id[:]), Path: lockRequest.Path, Owner: User{Name: "Git LFS Tests"}, LockedAt: time.Now(), } addLocks(repo, *lock) // TODO(taylor): commit_needed case // TODO(taylor): err case enc.Encode(&LockResponse{ Lock: lock, }) return } } http.NotFound(w, r) } func missingRequiredCreds(w http.ResponseWriter, r *http.Request, repo string) bool { if !strings.HasPrefix(repo, "requirecreds") { return false } auth := r.Header.Get("Authorization") if len(auth) == 0 { writeLFSError(w, 401, "Error: Authorization Required") return true } user, pass, err := extractAuth(auth) if err != nil { writeLFSError(w, 403, err.Error()) return true } if user != "requirecreds" || pass != "pass" { writeLFSError(w, 403, fmt.Sprintf("Got: '%s' => '%s' : '%s'", auth, user, pass)) return true } return false } func testingChunkedTransferEncoding(r *http.Request) bool { return strings.HasPrefix(r.URL.String(), "/test-chunked-transfer-encoding") } func testingTusUploadInBatchReq(r *http.Request) bool { return strings.HasPrefix(r.URL.String(), "/test-tus-upload") } func testingTusUploadInterruptedInBatchReq(r *http.Request) bool { return strings.HasPrefix(r.URL.String(), "/test-tus-upload-interrupt") } func testingCustomTransfer(r *http.Request) bool { return strings.HasPrefix(r.URL.String(), "/test-custom-transfer") } var lfsUrlRE = regexp.MustCompile(`\A/?([^/]+)/info/lfs`) func repoFromLfsUrl(urlpath string) (string, error) { matches := lfsUrlRE.FindStringSubmatch(urlpath) if len(matches) != 2 { return "", fmt.Errorf("LFS url '%s' does not match %v", urlpath, lfsUrlRE) } repo := matches[1] if strings.HasSuffix(repo, ".git") { return repo[0 : len(repo)-4], nil } return repo, nil } type lfsStorage struct { objects map[string]map[string][]byte incomplete map[string]map[string][]byte mutex *sync.Mutex } func (s *lfsStorage) Get(repo, oid string) ([]byte, bool) { s.mutex.Lock() defer s.mutex.Unlock() repoObjects, ok := s.objects[repo] if !ok { return nil, ok } by, ok := repoObjects[oid] return by, ok } func (s *lfsStorage) Has(repo, oid string) bool { s.mutex.Lock() defer s.mutex.Unlock() repoObjects, ok := s.objects[repo] if !ok { return false } _, ok = repoObjects[oid] return ok } func (s *lfsStorage) Set(repo, oid string, by []byte) { s.mutex.Lock() defer s.mutex.Unlock() repoObjects, ok := s.objects[repo] if !ok { repoObjects = make(map[string][]byte) s.objects[repo] = repoObjects } repoObjects[oid] = by } func (s *lfsStorage) Delete(repo, oid string) { s.mutex.Lock() defer s.mutex.Unlock() repoObjects, ok := s.objects[repo] if ok { delete(repoObjects, oid) } } func (s *lfsStorage) GetIncomplete(repo, oid string) ([]byte, bool) { s.mutex.Lock() defer s.mutex.Unlock() repoObjects, ok := s.incomplete[repo] if !ok { return nil, ok } by, ok := repoObjects[oid] return by, ok } func (s *lfsStorage) SetIncomplete(repo, oid string, by []byte) { s.mutex.Lock() defer s.mutex.Unlock() repoObjects, ok := s.incomplete[repo] if !ok { repoObjects = make(map[string][]byte) s.incomplete[repo] = repoObjects } repoObjects[oid] = by } func (s *lfsStorage) DeleteIncomplete(repo, oid string) { s.mutex.Lock() defer s.mutex.Unlock() repoObjects, ok := s.incomplete[repo] if ok { delete(repoObjects, oid) } } func newLfsStorage() *lfsStorage { return &lfsStorage{ objects: make(map[string]map[string][]byte), incomplete: make(map[string]map[string][]byte), mutex: &sync.Mutex{}, } } func extractAuth(auth string) (string, string, error) { if strings.HasPrefix(auth, "Basic ") { decodeBy, err := base64.StdEncoding.DecodeString(auth[6:len(auth)]) decoded := string(decodeBy) if err != nil { return "", "", err } parts := strings.SplitN(decoded, ":", 2) if len(parts) == 2 { return parts[0], parts[1], nil } return "", "", nil } return "", "", nil } func skipIfNoCookie(w http.ResponseWriter, r *http.Request, id string) bool { cookie := r.Header.Get("Cookie") if strings.Contains(cookie, "secret") { return false } w.WriteHeader(403) debug(id, "No cookie received: %q", r.URL.Path) return true } func skipIfBadAuth(w http.ResponseWriter, r *http.Request, id string) bool { auth := r.Header.Get("Authorization") if auth == "" { w.WriteHeader(401) return true } user, pass, err := extractAuth(auth) if err != nil { w.WriteHeader(403) debug(id, "Error decoding auth: %s", err) return true } switch user { case "user": if pass == "pass" { return false } case "netrcuser", "requirecreds": return false case "path": if strings.HasPrefix(r.URL.Path, "/"+pass) { return false } debug(id, "auth attempt against: %q", r.URL.Path) } w.WriteHeader(403) debug(id, "Bad auth: %q", auth) return true } func init() { oidHandlers = make(map[string]string) for _, content := range contentHandlers { h := sha256.New() h.Write([]byte(content)) oidHandlers[hex.EncodeToString(h.Sum(nil))] = content } } func debug(reqid, msg string, args ...interface{}) { fullargs := make([]interface{}, len(args)+1) fullargs[0] = reqid for i, a := range args { fullargs[i+1] = a } log.Printf("[%s] "+msg+"\n", fullargs...) } func reqId(w http.ResponseWriter) (string, bool) { b := make([]byte, 16) _, err := rand.Read(b) if err != nil { http.Error(w, "error generating id: "+err.Error(), 500) return "", false } return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), true } // https://ericchiang.github.io/post/go-tls/ func generateCARootCertificates() (rootKey *rsa.PrivateKey, rootCert *x509.Certificate) { // generate a new key-pair rootKey, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { log.Fatalf("generating random key: %v", err) } rootCertTmpl, err := CertTemplate() if err != nil { log.Fatalf("creating cert template: %v", err) } // describe what the certificate will be used for rootCertTmpl.IsCA = true rootCertTmpl.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature rootCertTmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth} // rootCertTmpl.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")} rootCert, _, err = CreateCert(rootCertTmpl, rootCertTmpl, &rootKey.PublicKey, rootKey) return } func generateClientCertificates(rootCert *x509.Certificate, rootKey interface{}) (clientKey *rsa.PrivateKey, clientCertPEM []byte, clientKeyPEM []byte, clientKeyEncPEM []byte) { // create a key-pair for the client clientKey, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { log.Fatalf("generating random key: %v", err) } // create a template for the client clientCertTmpl, err1 := CertTemplate() if err1 != nil { log.Fatalf("creating cert template: %v", err1) } clientCertTmpl.KeyUsage = x509.KeyUsageDigitalSignature clientCertTmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} // the root cert signs the cert by again providing its private key _, clientCertPEM, err2 := CreateCert(clientCertTmpl, rootCert, &clientKey.PublicKey, rootKey) if err2 != nil { log.Fatalf("error creating cert: %v", err2) } privKey := x509.MarshalPKCS1PrivateKey(clientKey) // encode and load the cert and private key for the client clientKeyPEM = pem.EncodeToMemory(&pem.Block{ Type: "RSA PRIVATE KEY", Bytes: privKey, }) clientKeyEnc, err := x509.EncryptPEMBlock(bytes.NewBuffer(privKey), "RSA PRIVATE KEY", privKey, ([]byte)("pass"), x509.PEMCipherAES128) if err != nil { log.Fatalf("creating encrypted private key: %v", err) } clientKeyEncPEM = pem.EncodeToMemory(clientKeyEnc) return } // helper function to create a cert template with a serial number and other required fields func CertTemplate() (*x509.Certificate, error) { // generate a random serial number (a real cert authority would have some logic behind this) serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) if err != nil { return nil, errors.New("failed to generate serial number: " + err.Error()) } tmpl := x509.Certificate{ SerialNumber: serialNumber, Subject: pkix.Name{Organization: []string{"Yhat, Inc."}}, SignatureAlgorithm: x509.SHA256WithRSA, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), // valid for an hour BasicConstraintsValid: true, } return &tmpl, nil } func CreateCert(template, parent *x509.Certificate, pub interface{}, parentPriv interface{}) ( cert *x509.Certificate, certPEM []byte, err error) { certDER, err := x509.CreateCertificate(rand.Reader, template, parent, pub, parentPriv) if err != nil { return } // parse the resulting certificate so we can use it again cert, err = x509.ParseCertificate(certDER) if err != nil { return } // PEM encode the certificate (this is a standard TLS encoding) b := pem.Block{Type: "CERTIFICATE", Bytes: certDER} certPEM = pem.EncodeToMemory(&b) return }