Reduce integration test overhead (#32475)

In profiling integration tests, I found a couple places where per-test
overhead could be reduced:

* Avoiding disk IO by synchronizing instead of deleting & copying test
Git repository data. This saves ~100ms per test on my machine
* When flushing queues in `PrintCurrentTest`, invoke `FlushWithContext`
in a parallel.

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

models/fixtures/repository.yml

@@ -26,7 +26,7 @@
   fork_id: 0
   is_template: false
   template_id: 0
-  size: 8478
+  size: 0
   is_fsck_enabled: true
   close_issues_via_commit_in_any_branch: false
 

models/migrations/base/tests.go

@@ -8,7 +8,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path"
 	"path/filepath"
 	"runtime"
 	"testing"
@@ -35,27 +34,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...any) (*xorm.Engine, fu
 	ourSkip := 2
 	ourSkip += skip
 	deferFn := testlogger.PrintCurrentTest(t, ourSkip)
-	assert.NoError(t, os.RemoveAll(setting.RepoRootPath))
-	assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath))
-	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
-	if err != nil {
-		assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
-	}
-	for _, ownerDir := range ownerDirs {
-		if !ownerDir.Type().IsDir() {
-			continue
-		}
-		repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
-		if err != nil {
-			assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
-		}
-		for _, repoDir := range repoDirs {
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
-		}
-	}
+	assert.NoError(t, unittest.SyncDirs(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath))
 
 	if err := deleteDB(); err != nil {
 		t.Errorf("unable to reset database: %v", err)
@@ -123,7 +102,7 @@ func MainTest(m *testing.M) {
 	if runtime.GOOS == "windows" {
 		giteaBinary += ".exe"
 	}
-	setting.AppPath = path.Join(giteaRoot, giteaBinary)
+	setting.AppPath = filepath.Join(giteaRoot, giteaBinary)
 	if _, err := os.Stat(setting.AppPath); err != nil {
 		fmt.Printf("Could not find gitea binary at %s\n", setting.AppPath)
 		os.Exit(1)
@@ -131,12 +110,12 @@ func MainTest(m *testing.M) {
 
 	giteaConf := os.Getenv("GITEA_CONF")
 	if giteaConf == "" {
-		giteaConf = path.Join(filepath.Dir(setting.AppPath), "tests/sqlite.ini")
+		giteaConf = filepath.Join(filepath.Dir(setting.AppPath), "tests/sqlite.ini")
 		fmt.Printf("Environment variable $GITEA_CONF not set - defaulting to %s\n", giteaConf)
 	}
 
-	if !path.IsAbs(giteaConf) {
-		setting.CustomConf = path.Join(giteaRoot, giteaConf)
+	if !filepath.IsAbs(giteaConf) {
+		setting.CustomConf = filepath.Join(giteaRoot, giteaConf)
 	} else {
 		setting.CustomConf = giteaConf
 	}

models/unittest/fscopy.go

@@ -4,10 +4,8 @@
 package unittest
 
 import (
-	"errors"
-	"io"
 	"os"
-	"path"
+	"path/filepath"
 	"strings"
 
 	"code.gitea.io/gitea/modules/util"
@@ -32,67 +30,73 @@ func Copy(src, dest string) error {
 		return os.Symlink(target, dest)
 	}
 
-	sr, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer sr.Close()
-
-	dw, err := os.Create(dest)
-	if err != nil {
-		return err
-	}
-	defer dw.Close()
-
-	if _, err = io.Copy(dw, sr); err != nil {
-		return err
-	}
-
-	// Set back file information.
-	if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil {
-		return err
-	}
-	return os.Chmod(dest, si.Mode())
+	return util.CopyFile(src, dest)
 }
 
-// CopyDir copy files recursively from source to target directory.
-//
-// The filter accepts a function that process the path info.
-// and should return true for need to filter.
-//
-// It returns error when error occurs in underlying functions.
-func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error {
-	// Check if target directory exists.
-	if _, err := os.Stat(destPath); !errors.Is(err, os.ErrNotExist) {
-		return util.NewAlreadyExistErrorf("file or directory already exists: %s", destPath)
+// Sync synchronizes the two files. This is skipped if both files
+// exist and the size, modtime, and mode match.
+func Sync(srcPath, destPath string) error {
+	dest, err := os.Stat(destPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return Copy(srcPath, destPath)
+		}
+		return err
 	}
 
+	src, err := os.Stat(srcPath)
+	if err != nil {
+		return err
+	}
+
+	if src.Size() == dest.Size() &&
+		src.ModTime() == dest.ModTime() &&
+		src.Mode() == dest.Mode() {
+		return nil
+	}
+
+	return Copy(srcPath, destPath)
+}
+
+// SyncDirs synchronizes files recursively from source to target directory.
+// It returns error when error occurs in underlying functions.
+func SyncDirs(srcPath, destPath string) error {
 	err := os.MkdirAll(destPath, os.ModePerm)
 	if err != nil {
 		return err
 	}
 
-	// Gather directory info.
-	infos, err := util.StatDir(srcPath, true)
+	// find and delete all untracked files
+	destFiles, err := util.StatDir(destPath, true)
 	if err != nil {
 		return err
 	}
-
-	var filter func(filePath string) bool
-	if len(filters) > 0 {
-		filter = filters[0]
+	for _, destFile := range destFiles {
+		destFilePath := filepath.Join(destPath, destFile)
+		if _, err = os.Stat(filepath.Join(srcPath, destFile)); err != nil {
+			if os.IsNotExist(err) {
+				// if src file does not exist, remove dest file
+				if err = os.RemoveAll(destFilePath); err != nil {
+					return err
+				}
+			} else {
+				return err
+			}
+		}
 	}
 
-	for _, info := range infos {
-		if filter != nil && filter(info) {
-			continue
-		}
-
-		curPath := path.Join(destPath, info)
-		if strings.HasSuffix(info, "/") {
-			err = os.MkdirAll(curPath, os.ModePerm)
+	// sync src files to dest
+	srcFiles, err := util.StatDir(srcPath, true)
+	if err != nil {
+		return err
+	}
+	for _, srcFile := range srcFiles {
+		destFilePath := filepath.Join(destPath, srcFile)
+		// util.StatDir appends a slash to the directory name
+		if strings.HasSuffix(srcFile, "/") {
+			err = os.MkdirAll(destFilePath, os.ModePerm)
 		} else {
-			err = Copy(path.Join(srcPath, info), curPath)
+			err = Sync(filepath.Join(srcPath, srcFile), destFilePath)
 		}
 		if err != nil {
 			return err

models/unittest/testdb.go

@@ -164,35 +164,13 @@ func MainTest(m *testing.M, testOpts ...*TestOptions) {
 	if err = storage.Init(); err != nil {
 		fatalTestError("storage.Init: %v\n", err)
 	}
-	if err = util.RemoveAll(repoRootPath); err != nil {
-		fatalTestError("util.RemoveAll: %v\n", err)
-	}
-	if err = CopyDir(filepath.Join(giteaRoot, "tests", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
-		fatalTestError("util.CopyDir: %v\n", err)
+	if err = SyncDirs(filepath.Join(giteaRoot, "tests", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
+		fatalTestError("util.SyncDirs: %v\n", err)
 	}
 
 	if err = git.InitFull(context.Background()); err != nil {
 		fatalTestError("git.Init: %v\n", err)
 	}
-	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
-	if err != nil {
-		fatalTestError("unable to read the new repo root: %v\n", err)
-	}
-	for _, ownerDir := range ownerDirs {
-		if !ownerDir.Type().IsDir() {
-			continue
-		}
-		repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
-		if err != nil {
-			fatalTestError("unable to read the new repo root: %v\n", err)
-		}
-		for _, repoDir := range repoDirs {
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
-		}
-	}
 
 	if len(testOpts) > 0 && testOpts[0].SetUp != nil {
 		if err := testOpts[0].SetUp(); err != nil {
@@ -255,24 +233,7 @@ func PrepareTestDatabase() error {
 // by tests that use the above MainTest(..) function.
 func PrepareTestEnv(t testing.TB) {
 	assert.NoError(t, PrepareTestDatabase())
-	assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
 	metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta")
-	assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath))
-	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
-	assert.NoError(t, err)
-	for _, ownerDir := range ownerDirs {
-		if !ownerDir.Type().IsDir() {
-			continue
-		}
-		repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
-		assert.NoError(t, err)
-		for _, repoDir := range repoDirs {
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
-		}
-	}
-
+	assert.NoError(t, SyncDirs(metaPath, setting.RepoRootPath))
 	base.SetupGiteaRoot() // Makes sure GITEA_ROOT is set
 }

modules/git/tests/repos/language_stats_repo/description

@@ -1 +0,0 @@
-Unnamed repository; edit this file 'description' to name the repository.

modules/git/tests/repos/language_stats_repo/info/exclude

@@ -1,6 +0,0 @@
-# git ls-files --others --exclude-from=.git/info/exclude
-# Lines that start with '#' are comments.
-# For a project mostly in C, the following would be a good set of
-# exclude patterns (uncomment them if you want to use them):
-# *.[oa]
-# *~

modules/git/tests/repos/repo1_bare/description

@@ -1 +0,0 @@
-Unnamed repository; edit this file 'description' to name the repository.

modules/git/tests/repos/repo1_bare/info/exclude

@@ -1,6 +0,0 @@
-# git ls-files --others --exclude-from=.git/info/exclude
-# Lines that start with '#' are comments.
-# For a project mostly in C, the following would be a good set of
-# exclude patterns (uncomment them if you want to use them):
-# *.[oa]
-# *~

modules/git/tests/repos/repo1_bare_sha256/description

@@ -1 +0,0 @@
-Unnamed repository; edit this file 'description' to name the repository.

modules/git/tests/repos/repo1_bare_sha256/info/exclude

@@ -1,6 +0,0 @@
-# git ls-files --others --exclude-from=.git/info/exclude
-# Lines that start with '#' are comments.
-# For a project mostly in C, the following would be a good set of
-# exclude patterns (uncomment them if you want to use them):
-# *.[oa]
-# *~

modules/git/tests/repos/repo2_empty/description

@@ -1 +0,0 @@
-Unnamed repository; edit this file 'description' to name the repository.

modules/git/tests/repos/repo2_empty/info/exclude

@@ -1,6 +0,0 @@
-# git ls-files --others --exclude-from=.git/info/exclude
-# Lines that start with '#' are comments.
-# For a project mostly in C, the following would be a good set of
-# exclude patterns (uncomment them if you want to use them):
-# *.[oa]
-# *~

modules/git/tests/repos/repo3_notes/description

@@ -1 +0,0 @@
-Unnamed repository; edit this file 'description' to name the repository.

modules/git/tests/repos/repo5_pulls/description

@@ -1 +0,0 @@
-Unnamed repository; edit this file 'description' to name the repository.

modules/git/tests/repos/repo5_pulls/info/exclude

@@ -1,6 +0,0 @@
-# git ls-files --others --exclude-from=.git/info/exclude
-# Lines that start with '#' are comments.
-# For a project mostly in C, the following would be a good set of
-# exclude patterns (uncomment them if you want to use them):
-# *.[oa]
-# *~

modules/git/tests/repos/repo5_pulls_sha256/description

@@ -1 +0,0 @@
-Unnamed repository; edit this file 'description' to name the repository.

modules/git/tests/repos/repo6_blame_sha256/description

@@ -1 +0,0 @@
-Unnamed repository; edit this file 'description' to name the repository.

modules/git/tests/repos/repo6_blame_sha256/info/exclude

@@ -1,6 +0,0 @@
-# git ls-files --others --exclude-from=.git/info/exclude
-# Lines that start with '#' are comments.
-# For a project mostly in C, the following would be a good set of
-# exclude patterns (uncomment them if you want to use them):
-# *.[oa]
-# *~

modules/git/tests/repos/repo6_merge_sha256/description

@@ -1 +0,0 @@
-Unnamed repository; edit this file 'description' to name the repository.

modules/git/tests/repos/repo6_merge_sha256/info/exclude

@@ -1,6 +0,0 @@
-# git ls-files --others --exclude-from=.git/info/exclude
-# Lines that start with '#' are comments.
-# For a project mostly in C, the following would be a good set of
-# exclude patterns (uncomment them if you want to use them):
-# *.[oa]
-# *~

modules/indexer/code/indexer_test.go

@@ -21,6 +21,7 @@ import (
 	_ "code.gitea.io/gitea/models/activities"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	_ "github.com/mattn/go-sqlite3"
 )
@@ -284,15 +285,11 @@ func TestBleveIndexAndSearch(t *testing.T) {
 	dir := t.TempDir()
 
 	idx := bleve.NewIndexer(dir)
-	_, err := idx.Init(context.Background())
-	if err != nil {
-		if idx != nil {
-			idx.Close()
-		}
-		assert.FailNow(t, "Unable to create bleve indexer Error: %v", err)
-	}
 	defer idx.Close()
 
+	_, err := idx.Init(context.Background())
+	require.NoError(t, err)
+
 	testIndexer("beleve", t, idx)
 }
 

modules/queue/manager.go

@@ -5,6 +5,7 @@ package queue
 
 import (
 	"context"
+	"errors"
 	"sync"
 	"time"
 
@@ -32,6 +33,7 @@ type ManagedWorkerPoolQueue interface {
 
 	// FlushWithContext tries to make the handler process all items in the queue synchronously.
 	// It is for testing purpose only. It's not designed to be used in a cluster.
+	// Negative timeout means discarding all items in the queue.
 	FlushWithContext(ctx context.Context, timeout time.Duration) error
 
 	// RemoveAllItems removes all items in the base queue (on-the-fly items are not affected)
@@ -76,15 +78,16 @@ func (m *Manager) ManagedQueues() map[int64]ManagedWorkerPoolQueue {
 
 // FlushAll tries to make all managed queues process all items synchronously, until timeout or the queue is empty.
 // It is for testing purpose only. It's not designed to be used in a cluster.
+// Negative timeout means discarding all items in the queue.
 func (m *Manager) FlushAll(ctx context.Context, timeout time.Duration) error {
-	var finalErr error
+	var finalErrors []error
 	qs := m.ManagedQueues()
 	for _, q := range qs {
 		if err := q.FlushWithContext(ctx, timeout); err != nil {
-			finalErr = err // TODO: in Go 1.20: errors.Join
+			finalErrors = append(finalErrors, err)
 		}
 	}
-	return finalErr
+	return errors.Join(finalErrors...)
 }
 
 // CreateSimpleQueue creates a simple queue from global setting config provider by name

modules/queue/workergroup.go

@@ -197,15 +197,30 @@ func (q *WorkerPoolQueue[T]) doFlush(wg *workerGroup[T], flush flushType) {
 	defer log.Debug("Queue %q finishes flushing", q.GetName())
 
 	// stop all workers, and prepare a new worker context to start new workers
-
 	wg.ctxWorkerCancel()
 	wg.wg.Wait()
 
 	defer func() {
-		close(flush)
+		close(flush.c)
 		wg.doPrepareWorkerContext()
 	}()
 
+	if flush.timeout < 0 {
+		// discard everything
+		wg.batchBuffer = nil
+		for {
+			select {
+			case <-wg.popItemChan:
+			case <-wg.popItemErr:
+			case <-q.batchChan:
+			case <-q.ctxRun.Done():
+				return
+			default:
+				return
+			}
+		}
+	}
+
 	// drain the batch channel first
 loop:
 	for {

modules/queue/workerqueue.go

@@ -42,7 +42,10 @@ type WorkerPoolQueue[T any] struct {
 	workerNumMu     sync.Mutex
 }
 
-type flushType chan struct{}
+type flushType struct {
+	timeout time.Duration
+	c       chan struct{}
+}
 
 var _ ManagedWorkerPoolQueue = (*WorkerPoolQueue[any])(nil)
 
@@ -104,12 +107,12 @@ func (q *WorkerPoolQueue[T]) FlushWithContext(ctx context.Context, timeout time.
 	if timeout > 0 {
 		after = time.After(timeout)
 	}
-	c := make(flushType)
+	flush := flushType{timeout: timeout, c: make(chan struct{})}
 
 	// send flush request
 	// if it blocks, it means that there is a flush in progress or the queue hasn't been started yet
 	select {
-	case q.flushChan <- c:
+	case q.flushChan <- flush:
 	case <-ctx.Done():
 		return ctx.Err()
 	case <-q.ctxRun.Done():
@@ -120,7 +123,7 @@ func (q *WorkerPoolQueue[T]) FlushWithContext(ctx context.Context, timeout time.
 
 	// wait for flush to finish
 	select {
-	case <-c:
+	case <-flush.c:
 		return nil
 	case <-ctx.Done():
 		return ctx.Err()

modules/repository/create_test.go

@@ -38,8 +38,8 @@ func TestGetDirectorySize(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 	repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 1)
 	assert.NoError(t, err)
-
 	size, err := getDirectorySize(repo.RepoPath())
 	assert.NoError(t, err)
-	assert.EqualValues(t, size, repo.Size)
+	repo.Size = 8165 // real size on the disk
+	assert.EqualValues(t, repo.Size, size)
 }

modules/testlogger/testlogger.go

@@ -118,7 +118,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() {
 				_, _ = fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), SlowFlush)
 			}
 		})
-		if err := queue.GetManager().FlushAll(context.Background(), time.Minute); err != nil {
+		if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil {
 			t.Errorf("Flushing queues failed with error %v", err)
 		}
 		timer.Stop()

services/gitdiff/testdata/academic-module/description

@@ -1 +0,0 @@
-Unnamed repository; edit this file 'description' to name the repository.

services/gitdiff/testdata/academic-module/info/exclude

@@ -1,6 +0,0 @@
-# git ls-files --others --exclude-from=.git/info/exclude
-# Lines that start with '#' are comments.
-# For a project mostly in C, the following would be a good set of
-# exclude patterns (uncomment them if you want to use them):
-# *.[oa]
-# *~

services/repository/adopt_test.go

@@ -89,7 +89,7 @@ func TestListUnadoptedRepositories_ListOptions(t *testing.T) {
 
 func TestAdoptRepository(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
-	assert.NoError(t, unittest.CopyDir(filepath.Join(setting.RepoRootPath, "user2", "repo1.git"), filepath.Join(setting.RepoRootPath, "user2", "test-adopt.git")))
+	assert.NoError(t, unittest.SyncDirs(filepath.Join(setting.RepoRootPath, "user2", "repo1.git"), filepath.Join(setting.RepoRootPath, "user2", "test-adopt.git")))
 	user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
 	_, err := AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"})
 	assert.NoError(t, err)

tests/gitea-repositories-meta/limited_org/private_repo_on_limited_org.git/description

@@ -1 +0,0 @@
-Unnamed repository; edit this file 'description' to name the repository.