From 07489d0405d9e6c6e8cd432712968f26cb04df2a Mon Sep 17 00:00:00 2001
From: Lunny Xiao <xiaolunwen@gmail.com>
Date: Wed, 8 Sep 2021 02:39:05 +0800
Subject: [PATCH] Fix storage Iterate bug and Add storage doctor to delete
 garbage attachments (#16971) (#16977)

* Fix storage Iterate bug and Add storage doctor to delete garbage attachments

* Close object when used
---
 cmd/doctor.go             |  1 -
 models/attachment.go      |  5 +++
 modules/doctor/storage.go | 76 +++++++++++++++++++++++++++++++++++++++
 modules/storage/minio.go  |  4 +--
 4 files changed, 83 insertions(+), 3 deletions(-)
 create mode 100644 modules/doctor/storage.go

diff --git a/cmd/doctor.go b/cmd/doctor.go
index 0152aebe39..2e79275434 100644
--- a/cmd/doctor.go
+++ b/cmd/doctor.go
@@ -124,7 +124,6 @@ func runRecreateTable(ctx *cli.Context) error {
 }
 
 func runDoctor(ctx *cli.Context) error {
-
 	// Silence the default loggers
 	log.DelNamedLogger("console")
 	log.DelNamedLogger(log.DEFAULT)
diff --git a/models/attachment.go b/models/attachment.go
index e12609f810..4e0ccba5a0 100644
--- a/models/attachment.go
+++ b/models/attachment.go
@@ -144,6 +144,11 @@ func GetAttachmentByUUID(uuid string) (*Attachment, error) {
 	return getAttachmentByUUID(x, uuid)
 }
 
+// ExistAttachmentsByUUID returns true if attachment is exist by given UUID
+func ExistAttachmentsByUUID(uuid string) (bool, error) {
+	return x.Where("`uuid`=?", uuid).Exist(new(Attachment))
+}
+
 // GetAttachmentByReleaseIDFileName returns attachment by given releaseId and fileName.
 func GetAttachmentByReleaseIDFileName(releaseID int64, fileName string) (*Attachment, error) {
 	return getAttachmentByReleaseIDFileName(x, releaseID, fileName)
diff --git a/modules/doctor/storage.go b/modules/doctor/storage.go
new file mode 100644
index 0000000000..cb06d66204
--- /dev/null
+++ b/modules/doctor/storage.go
@@ -0,0 +1,76 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package doctor
+
+import (
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/storage"
+)
+
+func checkAttachmentStorageFiles(logger log.Logger, autofix bool) error {
+	var total, garbageNum int
+	var deletePaths []string
+	if err := storage.Attachments.IterateObjects(func(p string, obj storage.Object) error {
+		defer obj.Close()
+
+		total++
+		stat, err := obj.Stat()
+		if err != nil {
+			return err
+		}
+		exist, err := models.ExistAttachmentsByUUID(stat.Name())
+		if err != nil {
+			return err
+		}
+		if !exist {
+			garbageNum++
+			if autofix {
+				deletePaths = append(deletePaths, p)
+			}
+		}
+		return nil
+	}); err != nil {
+		logger.Error("storage.Attachments.IterateObjects failed: %v", err)
+		return err
+	}
+
+	if garbageNum > 0 {
+		if autofix {
+			var deletedNum int
+			for _, p := range deletePaths {
+				if err := storage.Attachments.Delete(p); err != nil {
+					log.Error("Delete attachment %s failed: %v", p, err)
+				} else {
+					deletedNum++
+				}
+			}
+			logger.Info("%d missed information attachment detected, %d deleted.", garbageNum, deletedNum)
+		} else {
+			logger.Warn("Checked %d attachment, %d missed information.", total, garbageNum)
+		}
+	}
+	return nil
+}
+
+func checkStorageFiles(logger log.Logger, autofix bool) error {
+	if err := storage.Init(); err != nil {
+		logger.Error("storage.Init failed: %v", err)
+		return err
+	}
+	return checkAttachmentStorageFiles(logger, autofix)
+}
+
+func init() {
+	Register(&Check{
+		Title:                      "Check if there is garbage storage files",
+		Name:                       "storages",
+		IsDefault:                  false,
+		Run:                        checkStorageFiles,
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+}
diff --git a/modules/storage/minio.go b/modules/storage/minio.go
index 724445c0ab..f78ba6aa27 100644
--- a/modules/storage/minio.go
+++ b/modules/storage/minio.go
@@ -151,7 +151,7 @@ type minioFileInfo struct {
 }
 
 func (m minioFileInfo) Name() string {
-	return m.ObjectInfo.Key
+	return path.Base(m.ObjectInfo.Key)
 }
 
 func (m minioFileInfo) Size() int64 {
@@ -219,7 +219,7 @@ func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) er
 		}
 		if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
 			defer object.Close()
-			return fn(strings.TrimPrefix(m.basePath, mObjInfo.Key), &minioObject{object})
+			return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object})
 		}(object, fn); err != nil {
 			return convertMinioErr(err)
 		}