lianguan/gitea
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move from max( id ) to max( index ) for latest commit statuses (#30076) (#30155)

Browse Source 
Backport https://github.com/go-gitea/gitea/pull/30076.

This PR replaces the use of `max( id )`, and instead using ``max(
`index` )`` for determining the latest commit status. Building business
logic over an `auto_increment` primary key like `id` is risky and
there’re already plenty of discussions on the Internet.

There‘s no guarantee for `auto_increment` values to be monotonic,
especially upon failures or with a cluster. In the specific case, we met
the problem of commit statuses being outdated when using TiDB as the
database. As [being
documented](https://docs.pingcap.com/tidb/stable/auto-increment),
`auto_increment` values assigned to an `insert` statement will only be
monotonic on a per server (node) basis.

Closes #30074.
This commit is contained in:
YR Chen 2024-03-29 00:29:38 +08:00 committed by GitHub
parent 7bffb923ce
commit b1dae9f2c8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 73 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
models/git/commit_status.go
View File

@ -25,6 +25,7 @@ import (
"code.gitea.io/gitea/modules/translation"
"xorm.io/builder"
"xorm.io/xorm"
)
// CommitStatus holds a single Status of a single Commit
@ -281,44 +282,48 @@ type CommitStatusIndex struct {
// GetLatestCommitStatus returns all statuses with a unique context for a given commit.
func GetLatestCommitStatus(ctx context.Context, repoID int64, sha string, listOptions db.ListOptions) ([]*CommitStatus, int64, error) {
ids := make([]int64, 0, 10)
sess := db.GetEngine(ctx).Table(&CommitStatus{}).
Where("repo_id = ?", repoID).And("sha = ?", sha).
Select("max( id ) as id").
GroupBy("context_hash").OrderBy("max( id ) desc")
getBase := func() *xorm.Session {
return db.GetEngine(ctx).Table(&CommitStatus{}).
Where("repo_id = ?", repoID).And("sha = ?", sha)
}
indices := make([]int64, 0, 10)
sess := getBase().Select("max( `index` ) as `index`").
GroupBy("context_hash").OrderBy("max( `index` ) desc")
if !listOptions.IsListAll() {
sess = db.SetSessionPagination(sess, &listOptions)
}
count, err := sess.FindAndCount(&ids)
count, err := sess.FindAndCount(&indices)
if err != nil {
return nil, count, err
}
statuses := make([]*CommitStatus, 0, len(ids))
if len(ids) == 0 {
statuses := make([]*CommitStatus, 0, len(indices))
if len(indices) == 0 {
return statuses, count, nil
}
return statuses, count, db.GetEngine(ctx).In("id", ids).Find(&statuses)
return statuses, count, getBase().And(builder.In("`index`", indices)).Find(&statuses)
}
// GetLatestCommitStatusForPairs returns all statuses with a unique context for a given list of repo-sha pairs
func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHAs map[int64]string, listOptions db.ListOptions) (map[int64][]*CommitStatus, error) {
type result struct {
ID int64
Index int64
RepoID int64
}
results := make([]result, 0, len(repoIDsToLatestCommitSHAs))
sess := db.GetEngine(ctx).Table(&CommitStatus{})
getBase := func() *xorm.Session {
return db.GetEngine(ctx).Table(&CommitStatus{})
}
// Create a disjunction of conditions for each repoID and SHA pair
conds := make([]builder.Cond, 0, len(repoIDsToLatestCommitSHAs))
for repoID, sha := range repoIDsToLatestCommitSHAs {
conds = append(conds, builder.Eq{"repo_id": repoID, "sha": sha})
}
sess = sess.Where(builder.Or(conds...)).
Select("max( id ) as id, repo_id").
GroupBy("context_hash, repo_id").OrderBy("max( id ) desc")
sess := getBase().Where(builder.Or(conds...)).
Select("max( `index` ) as `index`, repo_id").
GroupBy("context_hash, repo_id").OrderBy("max( `index` ) desc")
sess = db.SetSessionPagination(sess, &listOptions)
@ -327,15 +332,21 @@ func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHA
return nil, err
}
ids := make([]int64, 0, len(results))
repoStatuses := make(map[int64][]*CommitStatus)
for _, result := range results {
ids = append(ids, result.ID)
}
statuses := make([]*CommitStatus, 0, len(ids))
if len(ids) > 0 {
err = db.GetEngine(ctx).In("id", ids).Find(&statuses)
if len(results) > 0 {
statuses := make([]*CommitStatus, 0, len(results))
conds = make([]builder.Cond, 0, len(results))
for _, result := range results {
cond := builder.Eq{
"`index`": result.Index,
"repo_id": result.RepoID,
"sha": repoIDsToLatestCommitSHAs[result.RepoID],
}
conds = append(conds, cond)
}
err = getBase().Where(builder.Or(conds...)).Find(&statuses)
if err != nil {
return nil, err
}
@ -352,42 +363,43 @@ func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHA
// GetLatestCommitStatusForRepoCommitIDs returns all statuses with a unique context for a given list of repo-sha pairs
func GetLatestCommitStatusForRepoCommitIDs(ctx context.Context, repoID int64, commitIDs []string) (map[string][]*CommitStatus, error) {
type result struct {
ID int64
Sha string
Index int64
SHA string
}
getBase := func() *xorm.Session {
return db.GetEngine(ctx).Table(&CommitStatus{}).Where("repo_id = ?", repoID)
}
results := make([]result, 0, len(commitIDs))
sess := db.GetEngine(ctx).Table(&CommitStatus{})
// Create a disjunction of conditions for each repoID and SHA pair
conds := make([]builder.Cond, 0, len(commitIDs))
for _, sha := range commitIDs {
conds = append(conds, builder.Eq{"sha": sha})
}
sess = sess.Where(builder.Eq{"repo_id": repoID}.And(builder.Or(conds...))).
Select("max( id ) as id, sha").
GroupBy("context_hash, sha").OrderBy("max( id ) desc")
sess := getBase().And(builder.Or(conds...)).
Select("max( `index` ) as `index`, sha").
GroupBy("context_hash, sha").OrderBy("max( `index` ) desc")
err := sess.Find(&results)
if err != nil {
return nil, err
}
ids := make([]int64, 0, len(results))
repoStatuses := make(map[string][]*CommitStatus)
for _, result := range results {
ids = append(ids, result.ID)
}
statuses := make([]*CommitStatus, 0, len(ids))
if len(ids) > 0 {
err = db.GetEngine(ctx).In("id", ids).Find(&statuses)
if len(results) > 0 {
statuses := make([]*CommitStatus, 0, len(results))
conds = make([]builder.Cond, 0, len(results))
for _, result := range results {
conds = append(conds, builder.Eq{"`index`": result.Index, "sha": result.SHA})
}
err = getBase().And(builder.Or(conds...)).Find(&statuses)
if err != nil {
return nil, err
}
// Group the statuses by repo ID
// Group the statuses by commit
for _, status := range statuses {
repoStatuses[status.SHA] = append(repoStatuses[status.SHA], status)
}
@ -398,22 +410,36 @@ func GetLatestCommitStatusForRepoCommitIDs(ctx context.Context, repoID int64, co
// FindRepoRecentCommitStatusContexts returns repository's recent commit status contexts
func FindRepoRecentCommitStatusContexts(ctx context.Context, repoID int64, before time.Duration) ([]string, error) {
type result struct {
Index int64
SHA string
}
getBase := func() *xorm.Session {
return db.GetEngine(ctx).Table(&CommitStatus{}).Where("repo_id = ?", repoID)
}
start := timeutil.TimeStampNow().AddDuration(-before)
ids := make([]int64, 0, 10)
if err := db.GetEngine(ctx).Table("commit_status").
Where("repo_id = ?", repoID).
And("updated_unix >= ?", start).
Select("max( id ) as id").
GroupBy("context_hash").OrderBy("max( id ) desc").
Find(&ids); err != nil {
results := make([]result, 0, 10)
sess := getBase().And("updated_unix >= ?", start).
Select("max( `index` ) as `index`, sha").
GroupBy("context_hash, sha").OrderBy("max( `index` ) desc")
err := sess.Find(&results)
if err != nil {
return nil, err
}
contexts := make([]string, 0, len(ids))
if len(ids) == 0 {
contexts := make([]string, 0, len(results))
if len(results) == 0 {
return contexts, nil
}
return contexts, db.GetEngine(ctx).Select("context").Table("commit_status").In("id", ids).Find(&contexts)
conds := make([]builder.Cond, 0, len(results))
for _, result := range results {
conds = append(conds, builder.Eq{"`index`": result.Index, "sha": result.SHA})
}
return contexts, getBase().And(builder.Or(conds...)).Select("context").Find(&contexts)
}
// NewCommitStatusOptions holds options for creating a CommitStatus