Recommend/convert to use case-sensitive collation for MySQL/MSSQL (#28662)
Mainly for MySQL/MSSQL. It is important for Gitea to use case-sensitive database charset collation. If the database is using a case-insensitive collation, Gitea will show startup error/warning messages, and show the errors/warnings on the admin panel's Self-Check page. Make `gitea doctor convert` work for MySQL to convert the collations of database & tables & columns. * Fix #28131 ## ⚠️ BREAKING ⚠️ It is not quite breaking, but it's highly recommended to convert the database&table&column to a consistent and case-sensitive collation.
This commit is contained in:
GitHub
parent
a80debc208
commit
2df7563f31
1
.gitignore
vendored
1
.gitignore
vendored
@ -11,6 +11,7 @@ _test
|
||||
.idea
|
||||
# Goland's output filename can not be set manually
|
||||
/go_build_*
|
||||
/gitea_*
|
||||
|
||||
# MS VSCode
|
||||
.vscode
|
||||
|
||||
@ -37,8 +37,8 @@ func runDoctorConvert(ctx *cli.Context) error {
|
||||
|
||||
switch {
|
||||
case setting.Database.Type.IsMySQL():
|
||||
if err := db.ConvertUtf8ToUtf8mb4(); err != nil {
|
||||
log.Fatal("Failed to convert database from utf8 to utf8mb4: %v", err)
|
||||
if err := db.ConvertDatabaseTable(); err != nil {
|
||||
log.Fatal("Failed to convert database & table: %v", err)
|
||||
return err
|
||||
}
|
||||
fmt.Println("Converted successfully, please confirm your database's character set is now utf8mb4")
|
||||
|
||||
@ -351,6 +351,7 @@ NAME = gitea
|
||||
USER = root
|
||||
;PASSWD = ;Use PASSWD = `your password` for quoting if you use special characters in the password.
|
||||
;SSL_MODE = false ; either "false" (default), "true", or "skip-verify"
|
||||
;CHARSET_COLLATION = ; Empty as default, Gitea will try to find a case-sensitive collation. Don't change it unless you clearly know what you need.
|
||||
;;
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;;
|
||||
@ -382,6 +383,7 @@ USER = root
|
||||
;NAME = gitea
|
||||
;USER = SA
|
||||
;PASSWD = MwantsaSecurePassword1
|
||||
;CHARSET_COLLATION = ; Empty as default, Gitea will try to find a case-sensitive collation. Don't change it unless you clearly know what you need.
|
||||
;;
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;;
|
||||
|
||||
@ -430,6 +430,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
|
||||
- `NAME`: **gitea**: Database name.
|
||||
- `USER`: **root**: Database username.
|
||||
- `PASSWD`: **_empty_**: Database user password. Use \`your password\` or """your password""" for quoting if you use special characters in the password.
|
||||
- `CHARSET_COLLATION`: **_empty_**: (MySQL/MSSQL only) Gitea expects to use a case-sensitive collation for database. Leave it empty to use the default collation decided by the Gitea. Don't change it unless you clearly know what you need.
|
||||
- `SCHEMA`: **_empty_**: For PostgreSQL only, schema to use if different from "public". The schema must exist beforehand,
|
||||
the user must have creation privileges on it, and the user search path must be set to the look into the schema first
|
||||
(e.g. `ALTER USER user SET SEARCH_PATH = schema_name,"$user",public;`).
|
||||
|
||||
@ -371,24 +371,6 @@ If you are receiving an error line containing `Error 1071: Specified key was too
|
||||
then you are attempting to run Gitea on tables which use the ISAM engine. While this may have worked by chance in previous versions of Gitea, it has never been officially supported and
|
||||
you must use InnoDB. You should run `ALTER TABLE table_name ENGINE=InnoDB;` for each table in the database.
|
||||
|
||||
If you are using MySQL 5, another possible fix is
|
||||
|
||||
```mysql
|
||||
SET GLOBAL innodb_file_format=Barracuda;
|
||||
SET GLOBAL innodb_file_per_table=1;
|
||||
SET GLOBAL innodb_large_prefix=1;
|
||||
```
|
||||
|
||||
## Why Are Emoji Broken On MySQL
|
||||
|
||||
Unfortunately MySQL's `utf8` charset does not completely allow all possible UTF-8 characters, in particular Emoji.
|
||||
They created a new charset and collation called `utf8mb4` that allows for emoji to be stored but tables which use
|
||||
the `utf8` charset, and connections which use the `utf8` charset will not use this.
|
||||
|
||||
Please run `gitea doctor convert`, or run `ALTER DATABASE database_name CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;`
|
||||
for the database_name and run `ALTER TABLE table_name CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;`
|
||||
for each table in the database.
|
||||
|
||||
## Why are Emoji displaying only as placeholders or in monochrome
|
||||
|
||||
Gitea requires the system or browser to have one of the supported Emoji fonts installed, which are Apple Color Emoji, Segoe UI Emoji, Segoe UI Symbol, Noto Color Emoji and Twemoji Mozilla. Generally, the operating system should already provide one of these fonts, but especially on Linux, it may be necessary to install them manually.
|
||||
|
||||
@ -375,25 +375,6 @@ Gitea 提供了一个子命令`gitea migrate`来初始化数据库，然后您
|
||||
的错误行，则表示您正在尝试在使用 ISAM 引擎的表上运行 Gitea。尽管在先前版本的 Gitea 中可能是凑巧能够工作的，但它从未得到官方支持，
|
||||
您必须使用 InnoDB。您应该对数据库中的每个表运行`ALTER TABLE table_name ENGINE=InnoDB;`。
|
||||
|
||||
如果您使用的是 MySQL 5，另一个可能的修复方法是：
|
||||
|
||||
```mysql
|
||||
SET GLOBAL innodb_file_format=Barracuda;
|
||||
SET GLOBAL innodb_file_per_table=1;
|
||||
SET GLOBAL innodb_large_prefix=1;
|
||||
```
|
||||
|
||||
## 为什么 MySQL 上的 Emoji 显示错误
|
||||
|
||||
不幸的是，MySQL 的`utf8`字符集不完全允许所有可能的 UTF-8 字符，特别是 Emoji。
|
||||
他们创建了一个名为 `utf8mb4`的字符集和校对规则，允许存储 Emoji，但使用
|
||||
utf8 字符集的表和连接将不会使用它。
|
||||
|
||||
请运行 `gitea doctor convert` 或对数据库运行 `ALTER DATABASE database_name CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;`
|
||||
并对每个表运行 `ALTER TABLE table_name CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;`。
|
||||
|
||||
您还需要将`app.ini`文件中的数据库字符集设置为`CHARSET=utf8mb4`。
|
||||
|
||||
## 为什么 Emoji 只显示占位符或单色图像
|
||||
|
||||
Gitea 需要系统或浏览器安装其中一个受支持的 Emoji 字体，例如 Apple Color Emoji、Segoe UI Emoji、Segoe UI Symbol、Noto Color Emoji 和 Twemoji Mozilla。通常，操作系统应该已经提供了其中一个字体，但特别是在 Linux 上，可能需要手动安装它们。
|
||||
|
||||
@ -61,10 +61,14 @@ Note: All steps below requires that the database engine of your choice is instal
|
||||
|
||||
Replace username and password above as appropriate.
|
||||
|
||||
4. Create database with UTF-8 charset and collation. Make sure to use `utf8mb4` charset instead of `utf8` as the former supports all Unicode characters (including emojis) beyond _Basic Multilingual Plane_. Also, collation chosen depending on your expected content. When in doubt, use either `unicode_ci` or `general_ci`.
|
||||
4. Create database with UTF-8 charset and case-sensitive collation.
|
||||
|
||||
`utf8mb4_bin` is a common collation for both MySQL/MariaDB.
|
||||
When Gitea starts, it will try to find a better collation (`utf8mb4_0900_as_cs` or `uca1400_as_cs`) and alter the database if it is possible.
|
||||
If you would like to use other collation, you can set `[database].CHARSET_COLLATION` in the `app.ini` file.
|
||||
|
||||
```sql
|
||||
CREATE DATABASE giteadb CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_unicode_ci';
|
||||
CREATE DATABASE giteadb CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_bin';
|
||||
```
|
||||
|
||||
Replace database name as appropriate.
|
||||
|
||||
@ -59,10 +59,12 @@ menu:
|
||||
|
||||
根据需要替换上述用户名和密码。
|
||||
|
||||
4. 使用 UTF-8 字符集和排序规则创建数据库。确保使用 `**utf8mb4**` 字符集，而不是 `utf8`，因为前者支持 _Basic Multilingual Plane_ 之外的所有 Unicode 字符（包括表情符号）。排序规则根据您预期的内容选择。如果不确定，可以使用 `unicode_ci` 或 `general_ci`。
|
||||
4. 使用 UTF-8 字符集和大小写敏感的排序规则创建数据库。
|
||||
|
||||
Gitea 启动后会尝试把数据库修改为更合适的字符集，如果你想指定自己的字符集规则，可以在 app.ini 中设置 `[database].CHARSET_COLLATION`。
|
||||
|
||||
```sql
|
||||
CREATE DATABASE giteadb CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_unicode_ci';
|
||||
CREATE DATABASE giteadb CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_bin';
|
||||
```
|
||||
|
||||
根据需要替换数据库名称。
|
||||
|
||||
2
go.mod
2
go.mod
@ -121,7 +121,7 @@ require (
|
||||
mvdan.cc/xurls/v2 v2.5.0
|
||||
strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
|
||||
xorm.io/builder v0.3.13
|
||||
xorm.io/xorm v1.3.6
|
||||
xorm.io/xorm v1.3.7-0.20240101024435-4992cba040fe
|
||||
)
|
||||
|
||||
require (
|
||||
|
||||
191
models/db/collation.go
Normal file
191
models/db/collation.go
Normal file
@ -0,0 +1,191 @@
|
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/container"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
|
||||
"xorm.io/xorm"
|
||||
"xorm.io/xorm/schemas"
|
||||
)
|
||||
|
||||
type CheckCollationsResult struct {
|
||||
ExpectedCollation string
|
||||
AvailableCollation container.Set[string]
|
||||
DatabaseCollation string
|
||||
IsCollationCaseSensitive func(s string) bool
|
||||
CollationEquals func(a, b string) bool
|
||||
ExistingTableNumber int
|
||||
|
||||
InconsistentCollationColumns []string
|
||||
}
|
||||
|
||||
func findAvailableCollationsMySQL(x *xorm.Engine) (ret container.Set[string], err error) {
|
||||
var res []struct {
|
||||
Collation string
|
||||
}
|
||||
if err = x.SQL("SHOW COLLATION WHERE (Collation = 'utf8mb4_bin') OR (Collation LIKE '%\\_as\\_cs%')").Find(&res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ret = make(container.Set[string], len(res))
|
||||
for _, r := range res {
|
||||
ret.Add(r.Collation)
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func findAvailableCollationsMSSQL(x *xorm.Engine) (ret container.Set[string], err error) {
|
||||
var res []struct {
|
||||
Name string
|
||||
}
|
||||
if err = x.SQL("SELECT * FROM sys.fn_helpcollations() WHERE name LIKE '%[_]CS[_]AS%'").Find(&res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ret = make(container.Set[string], len(res))
|
||||
for _, r := range res {
|
||||
ret.Add(r.Name)
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func CheckCollations(x *xorm.Engine) (*CheckCollationsResult, error) {
|
||||
dbTables, err := x.DBMetas()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := &CheckCollationsResult{
|
||||
ExistingTableNumber: len(dbTables),
|
||||
CollationEquals: func(a, b string) bool { return a == b },
|
||||
}
|
||||
|
||||
var candidateCollations []string
|
||||
if x.Dialect().URI().DBType == schemas.MYSQL {
|
||||
if _, err = x.SQL("SELECT @@collation_database").Get(&res.DatabaseCollation); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.IsCollationCaseSensitive = func(s string) bool {
|
||||
return s == "utf8mb4_bin" || strings.HasSuffix(s, "_as_cs")
|
||||
}
|
||||
candidateCollations = []string{"utf8mb4_0900_as_cs", "uca1400_as_cs", "utf8mb4_bin"}
|
||||
res.AvailableCollation, err = findAvailableCollationsMySQL(x)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.CollationEquals = func(a, b string) bool {
|
||||
// MariaDB adds the "utf8mb4_" prefix, eg: "utf8mb4_uca1400_as_cs", but not the name "uca1400_as_cs" in "SHOW COLLATION"
|
||||
// At the moment, it's safe to ignore the database difference, just trim the prefix and compare. It could be fixed easily if there is any problem in the future.
|
||||
return a == b || strings.TrimPrefix(a, "utf8mb4_") == strings.TrimPrefix(b, "utf8mb4_")
|
||||
}
|
||||
} else if x.Dialect().URI().DBType == schemas.MSSQL {
|
||||
if _, err = x.SQL("SELECT DATABASEPROPERTYEX(DB_NAME(), 'Collation')").Get(&res.DatabaseCollation); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.IsCollationCaseSensitive = func(s string) bool {
|
||||
return strings.HasSuffix(s, "_CS_AS")
|
||||
}
|
||||
candidateCollations = []string{"Latin1_General_CS_AS"}
|
||||
res.AvailableCollation, err = findAvailableCollationsMSSQL(x)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if res.DatabaseCollation == "" {
|
||||
return nil, errors.New("unable to get collation for current database")
|
||||
}
|
||||
|
||||
res.ExpectedCollation = setting.Database.CharsetCollation
|
||||
if res.ExpectedCollation == "" {
|
||||
for _, collation := range candidateCollations {
|
||||
if res.AvailableCollation.Contains(collation) {
|
||||
res.ExpectedCollation = collation
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if res.ExpectedCollation == "" {
|
||||
return nil, errors.New("unable to find a suitable collation for current database")
|
||||
}
|
||||
|
||||
allColumnsMatchExpected := true
|
||||
allColumnsMatchDatabase := true
|
||||
for _, table := range dbTables {
|
||||
for _, col := range table.Columns() {
|
||||
if col.Collation != "" {
|
||||
allColumnsMatchExpected = allColumnsMatchExpected && res.CollationEquals(col.Collation, res.ExpectedCollation)
|
||||
allColumnsMatchDatabase = allColumnsMatchDatabase && res.CollationEquals(col.Collation, res.DatabaseCollation)
|
||||
if !res.IsCollationCaseSensitive(col.Collation) || !res.CollationEquals(col.Collation, res.DatabaseCollation) {
|
||||
res.InconsistentCollationColumns = append(res.InconsistentCollationColumns, fmt.Sprintf("%s.%s", table.Name, col.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// if all columns match expected collation or all match database collation, then it could also be considered as "consistent"
|
||||
if allColumnsMatchExpected || allColumnsMatchDatabase {
|
||||
res.InconsistentCollationColumns = nil
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func CheckCollationsDefaultEngine() (*CheckCollationsResult, error) {
|
||||
return CheckCollations(x)
|
||||
}
|
||||
|
||||
func alterDatabaseCollation(x *xorm.Engine, collation string) error {
|
||||
if x.Dialect().URI().DBType == schemas.MYSQL {
|
||||
_, err := x.Exec("ALTER DATABASE CHARACTER SET utf8mb4 COLLATE " + collation)
|
||||
return err
|
||||
} else if x.Dialect().URI().DBType == schemas.MSSQL {
|
||||
// TODO: MSSQL has many limitations on changing database collation, it could fail in many cases.
|
||||
_, err := x.Exec("ALTER DATABASE CURRENT COLLATE " + collation)
|
||||
return err
|
||||
}
|
||||
return errors.New("unsupported database type")
|
||||
}
|
||||
|
||||
// preprocessDatabaseCollation checks database & table column collation, and alter the database collation if needed
|
||||
func preprocessDatabaseCollation(x *xorm.Engine) {
|
||||
r, err := CheckCollations(x)
|
||||
if err != nil {
|
||||
log.Error("Failed to check database collation: %v", err)
|
||||
}
|
||||
if r == nil {
|
||||
return // no check result means the database doesn't need to do such check/process (at the moment ....)
|
||||
}
|
||||
|
||||
// try to alter database collation to expected if the database is empty, it might fail in some cases (and it isn't necessary to succeed)
|
||||
// at the moment, there is no "altering" solution for MSSQL, site admin should manually change the database collation
|
||||
// and there is a bug https://github.com/go-testfixtures/testfixtures/pull/182 mssql: Invalid object name 'information_schema.tables'.
|
||||
if !r.CollationEquals(r.DatabaseCollation, r.ExpectedCollation) && r.ExistingTableNumber == 0 && x.Dialect().URI().DBType == schemas.MYSQL {
|
||||
if err = alterDatabaseCollation(x, r.ExpectedCollation); err != nil {
|
||||
log.Error("Failed to change database collation to %q: %v", r.ExpectedCollation, err)
|
||||
} else {
|
||||
_, _ = x.Exec("SELECT 1") // after "altering", MSSQL's session becomes invalid, so make a simple query to "refresh" the session
|
||||
if r, err = CheckCollations(x); err != nil {
|
||||
log.Error("Failed to check database collation again after altering: %v", err) // impossible case
|
||||
return
|
||||
}
|
||||
log.Warn("Current database has been altered to use collation %q", r.DatabaseCollation)
|
||||
}
|
||||
}
|
||||
|
||||
// check column collation, and show warning/error to end users -- no need to fatal, do not block the startup
|
||||
if !r.IsCollationCaseSensitive(r.DatabaseCollation) {
|
||||
log.Warn("Current database is using a case-insensitive collation %q, although Gitea could work with it, there might be some rare cases which don't work as expected.", r.DatabaseCollation)
|
||||
}
|
||||
|
||||
if len(r.InconsistentCollationColumns) > 0 {
|
||||
log.Error("There are %d table columns using inconsistent collation, they should use %q. Please go to admin panel Self Check page", len(r.InconsistentCollationColumns), r.DatabaseCollation)
|
||||
}
|
||||
}
|
||||
@ -14,13 +14,18 @@ import (
|
||||
"xorm.io/xorm/schemas"
|
||||
)
|
||||
|
||||
// ConvertUtf8ToUtf8mb4 converts database and tables from utf8 to utf8mb4 if it's mysql and set ROW_FORMAT=dynamic
|
||||
func ConvertUtf8ToUtf8mb4() error {
|
||||
// ConvertDatabaseTable converts database and tables from utf8 to utf8mb4 if it's mysql and set ROW_FORMAT=dynamic
|
||||
func ConvertDatabaseTable() error {
|
||||
if x.Dialect().URI().DBType != schemas.MYSQL {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := x.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci", setting.Database.Name))
|
||||
r, err := CheckCollations(x)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = x.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE %s", setting.Database.Name, r.ExpectedCollation))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -30,11 +35,11 @@ func ConvertUtf8ToUtf8mb4() error {
|
||||
return err
|
||||
}
|
||||
for _, table := range tables {
|
||||
if _, err := x.Exec(fmt.Sprintf("ALTER TABLE `%s` ROW_FORMAT=dynamic;", table.Name)); err != nil {
|
||||
if _, err := x.Exec(fmt.Sprintf("ALTER TABLE `%s` ROW_FORMAT=dynamic", table.Name)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := x.Exec(fmt.Sprintf("ALTER TABLE `%s` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;", table.Name)); err != nil {
|
||||
if _, err := x.Exec(fmt.Sprintf("ALTER TABLE `%s` CONVERT TO CHARACTER SET utf8mb4 COLLATE %s", table.Name, r.ExpectedCollation)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@ -182,6 +182,8 @@ func InitEngineWithMigration(ctx context.Context, migrateFunc func(*xorm.Engine)
|
||||
return err
|
||||
}
|
||||
|
||||
preprocessDatabaseCollation(x)
|
||||
|
||||
// We have to run migrateFunc here in case the user is re-running installation on a previously created DB.
|
||||
// If we do not then table schemas will be changed and there will be conflicts when the migrations run properly.
|
||||
//
|
||||
|
||||
@ -35,6 +35,7 @@ var (
|
||||
Path string
|
||||
LogSQL bool
|
||||
MysqlCharset string
|
||||
CharsetCollation string
|
||||
Timeout int // seconds
|
||||
SQLiteJournalMode string
|
||||
DBConnectRetries int
|
||||
@ -67,7 +68,7 @@ func loadDBSetting(rootCfg ConfigProvider) {
|
||||
}
|
||||
Database.Schema = sec.Key("SCHEMA").String()
|
||||
Database.SSLMode = sec.Key("SSL_MODE").MustString("disable")
|
||||
Database.MysqlCharset = sec.Key("MYSQL_CHARSET").MustString("utf8mb4") // do not document it, end users won't need it.
|
||||
Database.CharsetCollation = sec.Key("CHARSET_COLLATION").String()
|
||||
|
||||
Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db"))
|
||||
Database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
|
||||
@ -105,8 +106,8 @@ func DBConnStr() (string, error) {
|
||||
if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL
|
||||
tls = "false"
|
||||
}
|
||||
connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s",
|
||||
Database.User, Database.Passwd, connType, Database.Host, Database.Name, paramSep, Database.MysqlCharset, tls)
|
||||
connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%sparseTime=true&tls=%s",
|
||||
Database.User, Database.Passwd, connType, Database.Host, Database.Name, paramSep, tls)
|
||||
case "postgres":
|
||||
connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, Database.SSLMode)
|
||||
case "mssql":
|
||||
|
||||
@ -2691,6 +2691,7 @@ teams.invite.description = Please click the button below to join the team.
|
||||
|
||||
[admin]
|
||||
dashboard = Dashboard
|
||||
self_check = Self Check
|
||||
identity_access = Identity & Access
|
||||
users = User Accounts
|
||||
organizations = Organizations
|
||||
@ -3216,6 +3217,13 @@ notices.desc = Description
|
||||
notices.op = Op.
|
||||
notices.delete_success = The system notices have been deleted.
|
||||
|
||||
self_check.no_problem_found = No problem found yet.
|
||||
self_check.database_collation_mismatch = Expect database to use collation: %s
|
||||
self_check.database_collation_case_insensitive = Database is using a collation %s, which is an insensitive collation. Although Gitea could work with it, there might be some rare cases which don't work as expected.
|
||||
self_check.database_inconsistent_collation_columns = Database is using collation %s, but these columns are using mismatched collations. It might cause some unexpected problems.
|
||||
self_check.database_fix_mysql = For MySQL/MariaDB users, you could use the "gitea doctor convert" command to fix the collation problems, or you could also fix the problem by "ALTER ... COLLATE ..." SQLs manually.
|
||||
self_check.database_fix_mssql = For MSSQL users, you could only fix the problem by "ALTER ... COLLATE ..." SQLs manually at the moment.
|
||||
|
||||
[action]
|
||||
create_repo = created repository <a href="%s">%s</a>
|
||||
rename_repo = renamed repository from <code>%[1]s</code> to <a href="%[2]s">%[3]s</a>
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
activities_model "code.gitea.io/gitea/models/activities"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/modules/base"
|
||||
"code.gitea.io/gitea/modules/context"
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
@ -27,6 +28,7 @@ import (
|
||||
|
||||
const (
|
||||
tplDashboard base.TplName = "admin/dashboard"
|
||||
tplSelfCheck base.TplName = "admin/self_check"
|
||||
tplCron base.TplName = "admin/cron"
|
||||
tplQueue base.TplName = "admin/queue"
|
||||
tplStacktrace base.TplName = "admin/stacktrace"
|
||||
@ -172,6 +174,33 @@ func DashboardPost(ctx *context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
func SelfCheck(ctx *context.Context) {
|
||||
ctx.Data["PageIsAdminSelfCheck"] = true
|
||||
r, err := db.CheckCollationsDefaultEngine()
|
||||
if err != nil {
|
||||
ctx.Flash.Error(fmt.Sprintf("CheckCollationsDefaultEngine: %v", err), true)
|
||||
}
|
||||
|
||||
if r != nil {
|
||||
ctx.Data["DatabaseType"] = setting.Database.Type
|
||||
ctx.Data["DatabaseCheckResult"] = r
|
||||
hasProblem := false
|
||||
if !r.CollationEquals(r.DatabaseCollation, r.ExpectedCollation) {
|
||||
ctx.Data["DatabaseCheckCollationMismatch"] = true
|
||||
hasProblem = true
|
||||
}
|
||||
if !r.IsCollationCaseSensitive(r.DatabaseCollation) {
|
||||
ctx.Data["DatabaseCheckCollationCaseInsensitive"] = true
|
||||
hasProblem = true
|
||||
}
|
||||
ctx.Data["DatabaseCheckInconsistentCollationColumns"] = r.InconsistentCollationColumns
|
||||
hasProblem = hasProblem || len(r.InconsistentCollationColumns) > 0
|
||||
|
||||
ctx.Data["DatabaseCheckHasProblems"] = hasProblem
|
||||
}
|
||||
ctx.HTML(http.StatusOK, tplSelfCheck)
|
||||
}
|
||||
|
||||
func CronTasks(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("admin.monitor.cron")
|
||||
ctx.Data["PageIsAdminMonitorCron"] = true
|
||||
|
||||
@ -678,6 +678,8 @@ func registerRoutes(m *web.Route) {
|
||||
m.Get("", admin.Dashboard)
|
||||
m.Post("", web.Bind(forms.AdminDashboardForm{}), admin.DashboardPost)
|
||||
|
||||
m.Get("/self_check", admin.SelfCheck)
|
||||
|
||||
m.Group("/config", func() {
|
||||
m.Get("", admin.Config)
|
||||
m.Post("", admin.ChangeConfig)
|
||||
|
||||
@ -4,6 +4,9 @@
|
||||
<a class="{{if .PageIsAdminDashboard}}active {{end}}item" href="{{AppSubUrl}}/admin">
|
||||
{{ctx.Locale.Tr "admin.dashboard"}}
|
||||
</a>
|
||||
<a class="{{if .PageIsAdminSelfCheck}}active {{end}}item" href="{{AppSubUrl}}/admin/self_check">
|
||||
{{ctx.Locale.Tr "admin.self_check"}}
|
||||
</a>
|
||||
<details class="item toggleable-item" {{if or .PageIsAdminUsers .PageIsAdminEmails .PageIsAdminOrganizations .PageIsAdminAuthentications}}open{{end}}>
|
||||
<summary>{{ctx.Locale.Tr "admin.identity_access"}}</summary>
|
||||
<div class="menu">
|
||||
|
||||
36
templates/admin/self_check.tmpl
Normal file
36
templates/admin/self_check.tmpl
Normal file
@ -0,0 +1,36 @@
|
||||
{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin config")}}
|
||||
|
||||
<div class="admin-setting-content">
|
||||
<h4 class="ui top attached header">
|
||||
{{ctx.Locale.Tr "admin.self_check"}}
|
||||
</h4>
|
||||
<div class="ui attached segment">
|
||||
{{if .DatabaseCheckHasProblems}}
|
||||
{{if .DatabaseType.IsMySQL}}
|
||||
<div class="gt-p-3">{{ctx.Locale.Tr "admin.self_check.database_fix_mysql"}}</div>
|
||||
{{else if .DatabaseType.IsMSSQL}}
|
||||
<div class="gt-p-3">{{ctx.Locale.Tr "admin.self_check.database_fix_mssql"}}</div>
|
||||
{{end}}
|
||||
{{if .DatabaseCheckCollationMismatch}}
|
||||
<div class="ui red message">{{ctx.Locale.Tr "admin.self_check.database_collation_mismatch" .DatabaseCheckResult.ExpectedCollation}}</div>
|
||||
{{end}}
|
||||
{{if .DatabaseCheckCollationCaseInsensitive}}
|
||||
<div class="ui warning message">{{ctx.Locale.Tr "admin.self_check.database_collation_case_insensitive" .DatabaseCheckResult.DatabaseCollation}}</div>
|
||||
{{end}}
|
||||
{{if .DatabaseCheckInconsistentCollationColumns}}
|
||||
<div class="ui red message">
|
||||
{{ctx.Locale.Tr "admin.self_check.database_inconsistent_collation_columns" .DatabaseCheckResult.DatabaseCollation}}
|
||||
<ul class="gt-w-100">
|
||||
{{range .DatabaseCheckInconsistentCollationColumns}}
|
||||
<li>{{.}}</li>
|
||||
{{end}}
|
||||
</ul>
|
||||
</div>
|
||||
{{end}}
|
||||
{{else}}
|
||||
<div class="gt-p-3">{{ctx.Locale.Tr "admin.self_check.no_problem_found"}}</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{{template "admin/layout_footer" .}}
|
||||
@ -139,6 +139,8 @@ func testAPICreateBranches(t *testing.T, giteaURL *url.URL) {
|
||||
ExpectedHTTPStatus: http.StatusConflict,
|
||||
},
|
||||
// Trying to create from other branch (not default branch)
|
||||
// ps: it can't test the case-sensitive behavior here: the "BRANCH_2" can't be created by git on a case-insensitive filesystem, it makes the test fail quickly before the database code.
|
||||
// Suppose some users are running Gitea on a case-insensitive filesystem, it seems that it's unable to support case-sensitive branch names.
|
||||
{
|
||||
OldBranch: "new_branch_from_master_1",
|
||||
NewBranch: "branch_2",
|
||||
@ -150,10 +152,18 @@ func testAPICreateBranches(t *testing.T, giteaURL *url.URL) {
|
||||
NewBranch: "new_branch_from_non_existent",
|
||||
ExpectedHTTPStatus: http.StatusNotFound,
|
||||
},
|
||||
// Trying to create a branch with UTF8
|
||||
{
|
||||
OldBranch: "master",
|
||||
NewBranch: "test-👀",
|
||||
ExpectedHTTPStatus: http.StatusCreated,
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
session := ctx.Session
|
||||
testAPICreateBranch(t, session, "user2", "my-noo-repo", test.OldBranch, test.NewBranch, test.ExpectedHTTPStatus)
|
||||
t.Run(test.NewBranch, func(t *testing.T) {
|
||||
testAPICreateBranch(t, session, "user2", "my-noo-repo", test.OldBranch, test.NewBranch, test.ExpectedHTTPStatus)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -168,7 +178,7 @@ func testAPICreateBranch(t testing.TB, session *TestSession, user, repo, oldBran
|
||||
var branch api.Branch
|
||||
DecodeJSON(t, resp, &branch)
|
||||
|
||||
if status == http.StatusCreated {
|
||||
if resp.Result().StatusCode == http.StatusCreated {
|
||||
assert.EqualValues(t, newBranch, branch.Name)
|
||||
}
|
||||
|
||||
|
||||
123
tests/integration/db_collation_test.go
Normal file
123
tests/integration/db_collation_test.go
Normal file
@ -0,0 +1,123 @@
|
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package integration
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/test"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"xorm.io/xorm"
|
||||
)
|
||||
|
||||
type TestCollationTbl struct {
|
||||
ID int64
|
||||
Txt string `xorm:"VARCHAR(10) UNIQUE"`
|
||||
}
|
||||
|
||||
func TestDatabaseCollation(t *testing.T) {
|
||||
x := db.GetEngine(db.DefaultContext).(*xorm.Engine)
|
||||
|
||||
// there are blockers for MSSQL to use case-sensitive collation, see the comments in db/collation.go
|
||||
if setting.Database.Type.IsMSSQL() {
|
||||
t.Skip("there are blockers for MSSQL to use case-sensitive collation")
|
||||
return
|
||||
}
|
||||
|
||||
// all created tables should use case-sensitive collation by default
|
||||
_, _ = x.Exec("DROP TABLE IF EXISTS test_collation_tbl")
|
||||
err := x.Sync(&TestCollationTbl{})
|
||||
assert.NoError(t, err)
|
||||
_, _ = x.Exec("INSERT INTO test_collation_tbl (txt) VALUES ('main')")
|
||||
_, _ = x.Exec("INSERT INTO test_collation_tbl (txt) VALUES ('Main')") // case-sensitive, so it inserts a new row
|
||||
_, _ = x.Exec("INSERT INTO test_collation_tbl (txt) VALUES ('main')") // duplicate, so it doesn't insert
|
||||
cnt, err := x.Count(&TestCollationTbl{})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 2, cnt)
|
||||
_, _ = x.Exec("DROP TABLE IF EXISTS test_collation_tbl")
|
||||
|
||||
// by default, SQLite3 and PostgreSQL are using case-sensitive collations, but MySQL and MSSQL are not
|
||||
// the following tests are only for MySQL and MSSQL
|
||||
if !setting.Database.Type.IsMySQL() && !setting.Database.Type.IsMSSQL() {
|
||||
t.Skip("only MySQL and MSSQL requires the case-sensitive collation check at the moment")
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("Default startup makes database collation case-sensitive", func(t *testing.T) {
|
||||
r, err := db.CheckCollations(x)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, r.IsCollationCaseSensitive(r.DatabaseCollation))
|
||||
assert.True(t, r.CollationEquals(r.ExpectedCollation, r.DatabaseCollation))
|
||||
assert.NotEmpty(t, r.AvailableCollation)
|
||||
assert.Empty(t, r.InconsistentCollationColumns)
|
||||
|
||||
// and by the way test the helper functions
|
||||
if setting.Database.Type.IsMySQL() {
|
||||
assert.True(t, r.IsCollationCaseSensitive("utf8mb4_bin"))
|
||||
assert.True(t, r.IsCollationCaseSensitive("utf8mb4_xxx_as_cs"))
|
||||
assert.False(t, r.IsCollationCaseSensitive("utf8mb4_general_ci"))
|
||||
assert.True(t, r.CollationEquals("abc", "abc"))
|
||||
assert.True(t, r.CollationEquals("abc", "utf8mb4_abc"))
|
||||
assert.False(t, r.CollationEquals("utf8mb4_general_ci", "utf8mb4_unicode_ci"))
|
||||
} else if setting.Database.Type.IsMSSQL() {
|
||||
assert.True(t, r.IsCollationCaseSensitive("Latin1_General_CS_AS"))
|
||||
assert.False(t, r.IsCollationCaseSensitive("Latin1_General_CI_AS"))
|
||||
assert.True(t, r.CollationEquals("abc", "abc"))
|
||||
assert.False(t, r.CollationEquals("Latin1_General_CS_AS", "SQL_Latin1_General_CP1_CS_AS"))
|
||||
} else {
|
||||
assert.Fail(t, "unexpected database type")
|
||||
}
|
||||
})
|
||||
|
||||
if setting.Database.Type.IsMSSQL() {
|
||||
return // skip table converting tests because MSSQL doesn't have a simple solution at the moment
|
||||
}
|
||||
|
||||
t.Run("Convert tables to utf8mb4_bin", func(t *testing.T) {
|
||||
defer test.MockVariableValue(&setting.Database.CharsetCollation, "utf8mb4_bin")()
|
||||
assert.NoError(t, db.ConvertDatabaseTable())
|
||||
r, err := db.CheckCollations(x)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "utf8mb4_bin", r.DatabaseCollation)
|
||||
assert.True(t, r.CollationEquals(r.ExpectedCollation, r.DatabaseCollation))
|
||||
assert.Empty(t, r.InconsistentCollationColumns)
|
||||
|
||||
_, _ = x.Exec("DROP TABLE IF EXISTS test_tbl")
|
||||
_, err = x.Exec("CREATE TABLE test_tbl (txt varchar(10) COLLATE utf8mb4_unicode_ci NOT NULL)")
|
||||
assert.NoError(t, err)
|
||||
r, err = db.CheckCollations(x)
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, r.InconsistentCollationColumns, "test_tbl.txt")
|
||||
})
|
||||
|
||||
t.Run("Convert tables to utf8mb4_general_ci", func(t *testing.T) {
|
||||
defer test.MockVariableValue(&setting.Database.CharsetCollation, "utf8mb4_general_ci")()
|
||||
assert.NoError(t, db.ConvertDatabaseTable())
|
||||
r, err := db.CheckCollations(x)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "utf8mb4_general_ci", r.DatabaseCollation)
|
||||
assert.True(t, r.CollationEquals(r.ExpectedCollation, r.DatabaseCollation))
|
||||
assert.Empty(t, r.InconsistentCollationColumns)
|
||||
|
||||
_, _ = x.Exec("DROP TABLE IF EXISTS test_tbl")
|
||||
_, err = x.Exec("CREATE TABLE test_tbl (txt varchar(10) COLLATE utf8mb4_bin NOT NULL)")
|
||||
assert.NoError(t, err)
|
||||
r, err = db.CheckCollations(x)
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, r.InconsistentCollationColumns, "test_tbl.txt")
|
||||
})
|
||||
|
||||
t.Run("Convert tables to default case-sensitive collation", func(t *testing.T) {
|
||||
defer test.MockVariableValue(&setting.Database.CharsetCollation, "")()
|
||||
assert.NoError(t, db.ConvertDatabaseTable())
|
||||
r, err := db.CheckCollations(x)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, r.IsCollationCaseSensitive(r.DatabaseCollation))
|
||||
assert.True(t, r.CollationEquals(r.ExpectedCollation, r.DatabaseCollation))
|
||||
assert.Empty(t, r.InconsistentCollationColumns)
|
||||
})
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user