Pooled and buffered gzip implementation (#5722)

* Pooled and buffered gzip implementation

* Add test for gzip

* Add integration test

* Ensure lfs check within transaction

The previous code made it possible for a race condition to occur whereby a LFSMetaObject could be checked into the database twice. We should check if the LFSMetaObject is within the database and insert it if not in one transaction.

* Try to avoid primary key problem in postgres

The integration tests are being affected by
https://github.com/go-testfixtures/testfixtures/issues/39 if we set the
primary key high enough, keep a count of this and remove at the end of
each test we shouldn't be affected by this.

integrations/lfs_getobject_test.go

@@ -0,0 +1,129 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/gzip"
+	"code.gitea.io/gitea/modules/lfs"
+	"code.gitea.io/gitea/modules/setting"
+	"github.com/stretchr/testify/assert"
+
+	gzipp "github.com/klauspost/compress/gzip"
+)
+
+func GenerateLFSOid(content io.Reader) (string, error) {
+	h := sha256.New()
+	if _, err := io.Copy(h, content); err != nil {
+		return "", err
+	}
+	sum := h.Sum(nil)
+	return hex.EncodeToString(sum), nil
+}
+
+var lfsID = int64(20000)
+
+func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string {
+	oid, err := GenerateLFSOid(bytes.NewReader(*content))
+	assert.NoError(t, err)
+	var lfsMetaObject *models.LFSMetaObject
+
+	if setting.UsePostgreSQL {
+		lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID}
+	} else {
+		lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID}
+	}
+
+	lfsID = lfsID + 1
+	lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
+	assert.NoError(t, err)
+	contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
+	if !contentStore.Exists(lfsMetaObject) {
+		err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content))
+		assert.NoError(t, err)
+	}
+	return oid
+}
+
+func doLfs(t *testing.T, content *[]byte, expectGzip bool) {
+	prepareTestEnv(t)
+	repo, err := models.GetRepositoryByOwnerAndName("user2", "repo1")
+	assert.NoError(t, err)
+	oid := storeObjectInRepo(t, repo.ID, content)
+	defer repo.RemoveLFSMetaObjectByOid(oid)
+
+	session := loginUser(t, "user2")
+
+	// Request OID
+	req := NewRequest(t, "GET", "/user2/repo1.git/info/lfs/objects/"+oid+"/test")
+	req.Header.Set("Accept-Encoding", "gzip")
+	resp := session.MakeRequest(t, req, http.StatusOK)
+
+	contentEncoding := resp.Header().Get("Content-Encoding")
+	if !expectGzip || !setting.EnableGzip {
+		assert.NotContains(t, contentEncoding, "gzip")
+
+		result := resp.Body.Bytes()
+		assert.Equal(t, *content, result)
+	} else {
+		assert.Contains(t, contentEncoding, "gzip")
+		gzippReader, err := gzipp.NewReader(resp.Body)
+		assert.NoError(t, err)
+		result, err := ioutil.ReadAll(gzippReader)
+		assert.NoError(t, err)
+		assert.Equal(t, *content, result)
+	}
+
+}
+
+func TestGetLFSSmall(t *testing.T) {
+	content := []byte("A very small file\n")
+	doLfs(t, &content, false)
+}
+
+func TestGetLFSLarge(t *testing.T) {
+	content := make([]byte, gzip.MinSize*10)
+	for i := range content {
+		content[i] = byte(i % 256)
+	}
+	doLfs(t, &content, true)
+}
+
+func TestGetLFSGzip(t *testing.T) {
+	b := make([]byte, gzip.MinSize*10)
+	for i := range b {
+		b[i] = byte(i % 256)
+	}
+	outputBuffer := bytes.NewBuffer([]byte{})
+	gzippWriter := gzipp.NewWriter(outputBuffer)
+	gzippWriter.Write(b)
+	gzippWriter.Close()
+	content := outputBuffer.Bytes()
+	doLfs(t, &content, false)
+}
+
+func TestGetLFSZip(t *testing.T) {
+	b := make([]byte, gzip.MinSize*10)
+	for i := range b {
+		b[i] = byte(i % 256)
+	}
+	outputBuffer := bytes.NewBuffer([]byte{})
+	zipWriter := zip.NewWriter(outputBuffer)
+	fileWriter, err := zipWriter.Create("default")
+	assert.NoError(t, err)
+	fileWriter.Write(b)
+	zipWriter.Close()
+	content := outputBuffer.Bytes()
+	doLfs(t, &content, false)
+}

integrations/sqlite.ini

@@ -30,6 +30,7 @@ LFS_CONTENT_PATH = data/lfs-sqlite
 OFFLINE_MODE     = false
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
 APP_DATA_PATH    = integrations/gitea-integration-sqlite/data
+ENABLE_GZIP      = true
 
 [mailer]
 ENABLED = false

models/lfs.go

@@ -44,20 +44,20 @@ const (
 func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) {
 	var err error
 
-	has, err := x.Get(m)
+	sess := x.NewSession()
+	defer sess.Close()
+	if err = sess.Begin(); err != nil {
+		return nil, err
+	}
+
+	has, err := sess.Get(m)
 	if err != nil {
 		return nil, err
 	}
 
 	if has {
 		m.Existing = true
-		return m, nil
+		return m, sess.Commit()
-	}
-
-	sess := x.NewSession()
-	defer sess.Close()
-	if err = sess.Begin(); err != nil {
-		return nil, err
 	}
 
 	if _, err = sess.Insert(m); err != nil {

modules/gzip/gzip_test.go

@@ -0,0 +1,131 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package gzip
+
+import (
+	"archive/zip"
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	gzipp "github.com/klauspost/compress/gzip"
+	"github.com/stretchr/testify/assert"
+	macaron "gopkg.in/macaron.v1"
+)
+
+func setup(sampleResponse []byte) (*macaron.Macaron, *[]byte) {
+	m := macaron.New()
+	m.Use(Middleware())
+	m.Get("/", func() *[]byte { return &sampleResponse })
+	return m, &sampleResponse
+}
+
+func reqNoAcceptGzip(t *testing.T, m *macaron.Macaron, sampleResponse *[]byte) {
+	// Request without accept gzip: Should not gzip
+	resp := httptest.NewRecorder()
+	req, err := http.NewRequest("GET", "/", nil)
+	assert.NoError(t, err)
+	m.ServeHTTP(resp, req)
+
+	_, ok := resp.HeaderMap[contentEncodingHeader]
+	assert.False(t, ok)
+
+	contentEncoding := resp.Header().Get(contentEncodingHeader)
+	assert.NotContains(t, contentEncoding, "gzip")
+
+	result := resp.Body.Bytes()
+	assert.Equal(t, *sampleResponse, result)
+}
+
+func reqAcceptGzip(t *testing.T, m *macaron.Macaron, sampleResponse *[]byte, expectGzip bool) {
+	// Request without accept gzip: Should not gzip
+	resp := httptest.NewRecorder()
+	req, err := http.NewRequest("GET", "/", nil)
+	assert.NoError(t, err)
+	req.Header.Set(acceptEncodingHeader, "gzip")
+	m.ServeHTTP(resp, req)
+
+	_, ok := resp.HeaderMap[contentEncodingHeader]
+	assert.Equal(t, ok, expectGzip)
+
+	contentEncoding := resp.Header().Get(contentEncodingHeader)
+	if expectGzip {
+		assert.Contains(t, contentEncoding, "gzip")
+		gzippReader, err := gzipp.NewReader(resp.Body)
+		assert.NoError(t, err)
+		result, err := ioutil.ReadAll(gzippReader)
+		assert.NoError(t, err)
+		assert.Equal(t, *sampleResponse, result)
+	} else {
+		assert.NotContains(t, contentEncoding, "gzip")
+		result := resp.Body.Bytes()
+		assert.Equal(t, *sampleResponse, result)
+	}
+}
+
+func TestMiddlewareSmall(t *testing.T) {
+	m, sampleResponse := setup([]byte("Small response"))
+
+	reqNoAcceptGzip(t, m, sampleResponse)
+
+	reqAcceptGzip(t, m, sampleResponse, false)
+}
+
+func TestMiddlewareLarge(t *testing.T) {
+	b := make([]byte, MinSize+1)
+	for i := range b {
+		b[i] = byte(i % 256)
+	}
+	m, sampleResponse := setup(b)
+
+	reqNoAcceptGzip(t, m, sampleResponse)
+
+	// This should be gzipped as we accept gzip
+	reqAcceptGzip(t, m, sampleResponse, true)
+}
+
+func TestMiddlewareGzip(t *testing.T) {
+	b := make([]byte, MinSize*10)
+	for i := range b {
+		b[i] = byte(i % 256)
+	}
+	outputBuffer := bytes.NewBuffer([]byte{})
+	gzippWriter := gzipp.NewWriter(outputBuffer)
+	gzippWriter.Write(b)
+	gzippWriter.Flush()
+	gzippWriter.Close()
+	output := outputBuffer.Bytes()
+
+	m, sampleResponse := setup(output)
+
+	reqNoAcceptGzip(t, m, sampleResponse)
+
+	// This should not be gzipped even though we accept gzip
+	reqAcceptGzip(t, m, sampleResponse, false)
+}
+
+func TestMiddlewareZip(t *testing.T) {
+	b := make([]byte, MinSize*10)
+	for i := range b {
+		b[i] = byte(i % 256)
+	}
+	outputBuffer := bytes.NewBuffer([]byte{})
+	zipWriter := zip.NewWriter(outputBuffer)
+	fileWriter, err := zipWriter.Create("default")
+	assert.NoError(t, err)
+	fileWriter.Write(b)
+	//fileWriter.Close()
+	zipWriter.Close()
+	output := outputBuffer.Bytes()
+
+	m, sampleResponse := setup(output)
+
+	reqNoAcceptGzip(t, m, sampleResponse)
+
+	// This should not be gzipped even though we accept gzip
+	reqAcceptGzip(t, m, sampleResponse, false)
+}

routers/routes/routes.go

@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/auth"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/gzip"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/metrics"
@@ -36,7 +37,6 @@ import (
 	"github.com/go-macaron/cache"
 	"github.com/go-macaron/captcha"
 	"github.com/go-macaron/csrf"
-	"github.com/go-macaron/gzip"
 	"github.com/go-macaron/i18n"
 	"github.com/go-macaron/session"
 	"github.com/go-macaron/toolbox"
@@ -54,7 +54,7 @@ func NewMacaron() *macaron.Macaron {
 	}
 	m.Use(macaron.Recovery())
 	if setting.EnableGzip {
-		m.Use(gzip.Gziper())
+		m.Use(gzip.Middleware())
 	}
 	if setting.Protocol == setting.FCGI {
 		m.SetURLPrefix(setting.AppSubURL)