Refactor "dump" sub-command (#30240)

Major changes:

* Move some functions like "addReader" / "isSubDir" /
"addRecursiveExclude" to a separate package, and add tests
* Clarify the filename&dump type logic and add tests
* Clarify the logger behavior and remove FIXME comments

Co-authored-by: Giteabot <teabot@gitea.io>

modules/dump/dumper.go

@@ -0,0 +1,174 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package dump
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"github.com/mholt/archiver/v3"
+)
+
+var SupportedOutputTypes = []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"}
+
+// PrepareFileNameAndType prepares the output file name and type, if the type is not supported, it returns an empty "outType"
+func PrepareFileNameAndType(argFile, argType string) (outFileName, outType string) {
+	if argFile == "" && argType == "" {
+		outType = SupportedOutputTypes[0]
+		outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType)
+	} else if argFile == "" {
+		outType = argType
+		outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType)
+	} else if argType == "" {
+		if filepath.Ext(outFileName) == "" {
+			outType = SupportedOutputTypes[0]
+			outFileName = argFile
+		} else {
+			for _, t := range SupportedOutputTypes {
+				if strings.HasSuffix(argFile, "."+t) {
+					outFileName = argFile
+					outType = t
+				}
+			}
+		}
+	} else {
+		outFileName, outType = argFile, argType
+	}
+	if !slices.Contains(SupportedOutputTypes, outType) {
+		return "", ""
+	}
+	return outFileName, outType
+}
+
+func IsSubdir(upper, lower string) (bool, error) {
+	if relPath, err := filepath.Rel(upper, lower); err != nil {
+		return false, err
+	} else if relPath == "." || !strings.HasPrefix(relPath, ".") {
+		return true, nil
+	}
+	return false, nil
+}
+
+type Dumper struct {
+	Writer  archiver.Writer
+	Verbose bool
+
+	globalExcludeAbsPaths []string
+}
+
+func (dumper *Dumper) AddReader(r io.ReadCloser, info os.FileInfo, customName string) error {
+	if dumper.Verbose {
+		log.Info("Adding file %s", customName)
+	}
+
+	return dumper.Writer.Write(archiver.File{
+		FileInfo: archiver.FileInfo{
+			FileInfo:   info,
+			CustomName: customName,
+		},
+		ReadCloser: r,
+	})
+}
+
+func (dumper *Dumper) AddFile(filePath, absPath string) error {
+	file, err := os.Open(absPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return err
+	}
+	return dumper.AddReader(file, fileInfo, filePath)
+}
+
+func (dumper *Dumper) normalizeFilePath(absPath string) string {
+	absPath = filepath.Clean(absPath)
+	if setting.IsWindows {
+		absPath = strings.ToLower(absPath)
+	}
+	return absPath
+}
+
+func (dumper *Dumper) GlobalExcludeAbsPath(absPaths ...string) {
+	for _, absPath := range absPaths {
+		dumper.globalExcludeAbsPaths = append(dumper.globalExcludeAbsPaths, dumper.normalizeFilePath(absPath))
+	}
+}
+
+func (dumper *Dumper) shouldExclude(absPath string, excludes []string) bool {
+	norm := dumper.normalizeFilePath(absPath)
+	return slices.Contains(dumper.globalExcludeAbsPaths, norm) || slices.Contains(excludes, norm)
+}
+
+func (dumper *Dumper) AddRecursiveExclude(insidePath, absPath string, excludes []string) error {
+	excludes = slices.Clone(excludes)
+	for i := range excludes {
+		excludes[i] = dumper.normalizeFilePath(excludes[i])
+	}
+	return dumper.addFileOrDir(insidePath, absPath, excludes)
+}
+
+func (dumper *Dumper) addFileOrDir(insidePath, absPath string, excludes []string) error {
+	absPath, err := filepath.Abs(absPath)
+	if err != nil {
+		return err
+	}
+	dir, err := os.Open(absPath)
+	if err != nil {
+		return err
+	}
+	defer dir.Close()
+
+	files, err := dir.Readdir(0)
+	if err != nil {
+		return err
+	}
+	for _, file := range files {
+		currentAbsPath := filepath.Join(absPath, file.Name())
+		if dumper.shouldExclude(currentAbsPath, excludes) {
+			continue
+		}
+
+		currentInsidePath := path.Join(insidePath, file.Name())
+		if file.IsDir() {
+			if err := dumper.AddFile(currentInsidePath, currentAbsPath); err != nil {
+				return err
+			}
+			if err = dumper.addFileOrDir(currentInsidePath, currentAbsPath, excludes); err != nil {
+				return err
+			}
+		} else {
+			// only copy regular files and symlink regular files, skip non-regular files like socket/pipe/...
+			shouldAdd := file.Mode().IsRegular()
+			if !shouldAdd && file.Mode()&os.ModeSymlink == os.ModeSymlink {
+				target, err := filepath.EvalSymlinks(currentAbsPath)
+				if err != nil {
+					return err
+				}
+				targetStat, err := os.Stat(target)
+				if err != nil {
+					return err
+				}
+				shouldAdd = targetStat.Mode().IsRegular()
+			}
+			if shouldAdd {
+				if err = dumper.AddFile(currentInsidePath, currentAbsPath); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}

modules/dump/dumper_test.go

@@ -0,0 +1,113 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package dump
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"github.com/mholt/archiver/v3"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPrepareFileNameAndType(t *testing.T) {
+	defer timeutil.MockSet(time.Unix(1234, 0))()
+	test := func(argFile, argType, expFile, expType string) {
+		outFile, outType := PrepareFileNameAndType(argFile, argType)
+		assert.Equal(t,
+			fmt.Sprintf("outFile=%s, outType=%s", expFile, expType),
+			fmt.Sprintf("outFile=%s, outType=%s", outFile, outType),
+			fmt.Sprintf("argFile=%s, argType=%s", argFile, argType),
+		)
+	}
+
+	test("", "", "gitea-dump-1234.zip", "zip")
+	test("", "tar.gz", "gitea-dump-1234.tar.gz", "tar.gz")
+	test("", "no-such", "", "")
+
+	test("-", "", "-", "zip")
+	test("-", "tar.gz", "-", "tar.gz")
+	test("-", "no-such", "", "")
+
+	test("a", "", "a", "zip")
+	test("a", "tar.gz", "a", "tar.gz")
+	test("a", "no-such", "", "")
+
+	test("a.zip", "", "a.zip", "zip")
+	test("a.zip", "tar.gz", "a.zip", "tar.gz")
+	test("a.zip", "no-such", "", "")
+
+	test("a.tar.gz", "", "a.tar.gz", "zip")
+	test("a.tar.gz", "tar.gz", "a.tar.gz", "tar.gz")
+	test("a.tar.gz", "no-such", "", "")
+}
+
+func TestIsSubDir(t *testing.T) {
+	tmpDir := t.TempDir()
+	_ = os.MkdirAll(filepath.Join(tmpDir, "include/sub"), 0o755)
+
+	isSub, err := IsSubdir(filepath.Join(tmpDir, "include"), filepath.Join(tmpDir, "include"))
+	assert.NoError(t, err)
+	assert.True(t, isSub)
+
+	isSub, err = IsSubdir(filepath.Join(tmpDir, "include"), filepath.Join(tmpDir, "include/sub"))
+	assert.NoError(t, err)
+	assert.True(t, isSub)
+
+	isSub, err = IsSubdir(filepath.Join(tmpDir, "include/sub"), filepath.Join(tmpDir, "include"))
+	assert.NoError(t, err)
+	assert.False(t, isSub)
+}
+
+type testWriter struct {
+	added []string
+}
+
+func (t *testWriter) Create(out io.Writer) error {
+	return nil
+}
+
+func (t *testWriter) Write(f archiver.File) error {
+	t.added = append(t.added, f.Name())
+	return nil
+}
+
+func (t *testWriter) Close() error {
+	return nil
+}
+
+func TestDumper(t *testing.T) {
+	sortStrings := func(s []string) []string {
+		sort.Strings(s)
+		return s
+	}
+	tmpDir := t.TempDir()
+	_ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude1"), 0o755)
+	_ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude2"), 0o755)
+	_ = os.MkdirAll(filepath.Join(tmpDir, "include/sub"), 0o755)
+	_ = os.WriteFile(filepath.Join(tmpDir, "include/a"), nil, 0o644)
+	_ = os.WriteFile(filepath.Join(tmpDir, "include/sub/b"), nil, 0o644)
+	_ = os.WriteFile(filepath.Join(tmpDir, "include/exclude1/a-1"), nil, 0o644)
+	_ = os.WriteFile(filepath.Join(tmpDir, "include/exclude2/a-2"), nil, 0o644)
+
+	tw := &testWriter{}
+	d := &Dumper{Writer: tw}
+	d.GlobalExcludeAbsPath(filepath.Join(tmpDir, "include/exclude1"))
+	err := d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), []string{filepath.Join(tmpDir, "include/exclude2")})
+	assert.NoError(t, err)
+	assert.EqualValues(t, sortStrings([]string{"include/a", "include/sub", "include/sub/b"}), sortStrings(tw.added))
+
+	tw = &testWriter{}
+	d = &Dumper{Writer: tw}
+	err = d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), nil)
+	assert.NoError(t, err)
+	assert.EqualValues(t, sortStrings([]string{"include/exclude2", "include/exclude2/a-2", "include/a", "include/sub", "include/sub/b", "include/exclude1", "include/exclude1/a-1"}), sortStrings(tw.added))
+}

modules/setting/log.go

@@ -185,8 +185,13 @@ func InitLoggersForTest() {
 	initAllLoggers()
 }
 
+var initLoggerDisabled bool
+
 // initAllLoggers creates all the log services
 func initAllLoggers() {
+	if initLoggerDisabled {
+		return
+	}
 	initManagedLoggers(log.GetManager(), CfgProvider)
 
 	golog.SetFlags(0)
@@ -194,6 +199,10 @@ func initAllLoggers() {
 	golog.SetOutput(log.LoggerToWriter(log.GetLogger(log.DEFAULT).Info))
 }
 
+func DisableLoggerInit() {
+	initLoggerDisabled = true
+}
+
 func initManagedLoggers(manager *log.LoggerManager, cfg ConfigProvider) {
 	loadLogGlobalFrom(cfg)
 	prepareLoggerConfig(cfg)

modules/timeutil/timestamp.go

@@ -21,8 +21,9 @@ var (
 )
 
 // MockSet sets the time to a mocked time.Time
-func MockSet(now time.Time) {
+func MockSet(now time.Time) func() {
 	mockNow = now
+	return MockUnset
 }
 
 // MockUnset will unset the mocked time.Time

modules/util/util.go

@@ -213,6 +213,14 @@ func ToPointer[T any](val T) *T {
 	return &val
 }
 
+// Iif is an "inline-if", it returns "trueVal" if "condition" is true, otherwise "falseVal"
+func Iif[T comparable](condition bool, trueVal, falseVal T) T {
+	if condition {
+		return trueVal
+	}
+	return falseVal
+}
+
 // IfZero returns "def" if "v" is a zero value, otherwise "v"
 func IfZero[T comparable](v, def T) T {
 	var zero T