lianguan/gitea
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Update migrated repositories' issues/comments/prs poster id if user has a github external user saved (#7751)

Browse Source 
* update migrated issues/comments when login as github

* add get userid when migrating or login with github oauth2

* fix lint

* add migrations for repository service type

* fix build

* remove unnecessary dependencies on migrations

* add cron task to update migrations poster ids and fix posterid when migrating

* fix lint

* fix lint

* improve code

* fix lint

* improve code

* replace releases publish id to actual author id

* fix import

* fix bug

* fix lint

* fix rawdata definition

* fix some bugs

* fix error message
This commit is contained in:
Lunny Xiao
2019-10-14 14:10:42 +08:00
committed by Lauris BH
parent ba201aaa44
commit e3e44a59d0
21 changed files with 740 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
modules/migrations/base/downloader.go
View File

@ -5,6 +5,8 @@
package base
import "code.gitea.io/gitea/modules/structs"
// Downloader downloads the site repo informations
type Downloader interface {
GetRepoInfo() (*Repository, error)
@ -21,4 +23,5 @@ type Downloader interface {
type DownloaderFactory interface {
Match(opts MigrateOptions) (bool, error)
New(opts MigrateOptions) (Downloader, error)
GitServiceType() structs.GitServiceType
}

239
modules/migrations/gitea.go
View File

@ -34,15 +34,17 @@ var (
// GiteaLocalUploader implements an Uploader to gitea sites
type GiteaLocalUploader struct {
doer *models.User
repoOwner string
repoName string
repo *models.Repository
labels sync.Map
milestones sync.Map
issues sync.Map
gitRepo *git.Repository
prHeadCache map[string]struct{}
doer *models.User
repoOwner string
repoName string
repo *models.Repository
labels sync.Map
milestones sync.Map
issues sync.Map
gitRepo *git.Repository
prHeadCache map[string]struct{}
userMap map[int64]int64 // external user id mapping to user id
gitServiceType structs.GitServiceType
}
// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
@ -52,6 +54,7 @@ func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *Gitea
repoOwner: repoOwner,
repoName: repoName,
prHeadCache: make(map[string]struct{}),
userMap: make(map[int64]int64),
}
}
@ -109,13 +112,15 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
}
r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{
RepoName: g.repoName,
Description: repo.Description,
Mirror: repo.IsMirror,
CloneAddr: remoteAddr,
Private: repo.IsPrivate,
Wiki: opts.Wiki,
Releases: opts.Releases, // if didn't get releases, then sync them from tags
RepoName: g.repoName,
Description: repo.Description,
OriginalURL: repo.OriginalURL,
GitServiceType: opts.GitServiceType,
Mirror: repo.IsMirror,
CloneAddr: remoteAddr,
Private: repo.IsPrivate,
Wiki: opts.Wiki,
Releases: opts.Releases, // if didn't get releases, then sync them from tags
})
g.repo = r
@ -193,20 +198,38 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
var rels = make([]*models.Release, 0, len(releases))
for _, release := range releases {
var rel = models.Release{
RepoID: g.repo.ID,
PublisherID: g.doer.ID,
TagName: release.TagName,
LowerTagName: strings.ToLower(release.TagName),
Target: release.TargetCommitish,
Title: release.Name,
Sha1: release.TargetCommitish,
Note: release.Body,
IsDraft: release.Draft,
IsPrerelease: release.Prerelease,
IsTag: false,
CreatedUnix: timeutil.TimeStamp(release.Created.Unix()),
OriginalAuthor: release.PublisherName,
OriginalAuthorID: release.PublisherID,
RepoID: g.repo.ID,
TagName: release.TagName,
LowerTagName: strings.ToLower(release.TagName),
Target: release.TargetCommitish,
Title: release.Name,
Sha1: release.TargetCommitish,
Note: release.Body,
IsDraft: release.Draft,
IsPrerelease: release.Prerelease,
IsTag: false,
CreatedUnix: timeutil.TimeStamp(release.Created.Unix()),
}
userid, ok := g.userMap[release.PublisherID]
tp := g.gitServiceType.Name()
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", release.PublisherID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[release.PublisherID] = userid
}
}
if userid > 0 {
rel.PublisherID = userid
} else {
rel.PublisherID = g.doer.ID
rel.OriginalAuthor = release.PublisherName
rel.OriginalAuthorID = release.PublisherID
}
// calc NumCommits
@ -284,20 +307,39 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
}
var is = models.Issue{
RepoID: g.repo.ID,
Repo: g.repo,
Index: issue.Number,
PosterID: g.doer.ID,
OriginalAuthor: issue.PosterName,
OriginalAuthorID: issue.PosterID,
Title: issue.Title,
Content: issue.Content,
IsClosed: issue.State == "closed",
IsLocked: issue.IsLocked,
MilestoneID: milestoneID,
Labels: labels,
CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()),
RepoID: g.repo.ID,
Repo: g.repo,
Index: issue.Number,
Title: issue.Title,
Content: issue.Content,
IsClosed: issue.State == "closed",
IsLocked: issue.IsLocked,
MilestoneID: milestoneID,
Labels: labels,
CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()),
}
userid, ok := g.userMap[issue.PosterID]
tp := g.gitServiceType.Name()
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", issue.PosterID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[issue.PosterID] = userid
}
}
if userid > 0 {
is.PosterID = userid
} else {
is.PosterID = g.doer.ID
is.OriginalAuthor = issue.PosterName
is.OriginalAuthorID = issue.PosterID
}
if issue.Closed != nil {
is.ClosedUnix = timeutil.TimeStamp(issue.Closed.Unix())
}
@ -331,15 +373,35 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
issueID = issueIDStr.(int64)
}
cms = append(cms, &models.Comment{
IssueID: issueID,
Type: models.CommentTypeComment,
PosterID: g.doer.ID,
OriginalAuthor: comment.PosterName,
OriginalAuthorID: comment.PosterID,
Content: comment.Content,
CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()),
})
userid, ok := g.userMap[comment.PosterID]
tp := g.gitServiceType.Name()
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", comment.PosterID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[comment.PosterID] = userid
}
}
cm := models.Comment{
IssueID: issueID,
Type: models.CommentTypeComment,
Content: comment.Content,
CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()),
}
if userid > 0 {
cm.PosterID = userid
} else {
cm.PosterID = g.doer.ID
cm.OriginalAuthor = comment.PosterName
cm.OriginalAuthorID = comment.PosterID
}
cms = append(cms, &cm)
// TODO: Reactions
}
@ -355,6 +417,28 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error
if err != nil {
return err
}
userid, ok := g.userMap[pr.PosterID]
tp := g.gitServiceType.Name()
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[pr.PosterID] = userid
}
}
if userid > 0 {
gpr.Issue.PosterID = userid
} else {
gpr.Issue.PosterID = g.doer.ID
gpr.Issue.OriginalAuthor = pr.PosterName
gpr.Issue.OriginalAuthorID = pr.PosterID
}
gprs = append(gprs, gpr)
}
if err := models.InsertPullRequests(gprs...); err != nil {
@ -460,6 +544,40 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
head = pr.Head.Ref
}
var issue = models.Issue{
RepoID: g.repo.ID,
Repo: g.repo,
Title: pr.Title,
Index: pr.Number,
Content: pr.Content,
MilestoneID: milestoneID,
IsPull: true,
IsClosed: pr.State == "closed",
IsLocked: pr.IsLocked,
Labels: labels,
CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()),
}
userid, ok := g.userMap[pr.PosterID]
if !ok {
var err error
userid, err = models.GetUserIDByExternalUserID("github", fmt.Sprintf("%v", pr.PosterID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[pr.PosterID] = userid
}
}
if userid > 0 {
issue.PosterID = userid
} else {
issue.PosterID = g.doer.ID
issue.OriginalAuthor = pr.PosterName
issue.OriginalAuthorID = pr.PosterID
}
var pullRequest = models.PullRequest{
HeadRepoID: g.repo.ID,
HeadBranch: head,
@ -470,22 +588,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
Index: pr.Number,
HasMerged: pr.Merged,
Issue: &models.Issue{
RepoID: g.repo.ID,
Repo: g.repo,
Title: pr.Title,
Index: pr.Number,
PosterID: g.doer.ID,
OriginalAuthor: pr.PosterName,
OriginalAuthorID: pr.PosterID,
Content: pr.Content,
MilestoneID: milestoneID,
IsPull: true,
IsClosed: pr.State == "closed",
IsLocked: pr.IsLocked,
Labels: labels,
CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()),
},
Issue: &issue,
}
if pullRequest.Issue.IsClosed && pr.Closed != nil {

8
modules/migrations/github.go
View File

@ -14,6 +14,7 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/structs"
"github.com/google/go-github/v24/github"
"golang.org/x/oauth2"
@ -39,7 +40,7 @@ func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error
return false, err
}
return u.Host == "github.com" && opts.AuthUsername != "", nil
return strings.EqualFold(u.Host, "github.com") && opts.AuthUsername != "", nil
}
// New returns a Downloader related to this factory according MigrateOptions
@ -58,6 +59,11 @@ func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Download
return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
}
// GitServiceType returns the type of git service
func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
return structs.GithubService
}
// GithubDownloaderV3 implements a Downloader interface to get repository informations
// from github via APIv3
type GithubDownloaderV3 struct {

7
modules/migrations/migrate.go
View File

@ -11,6 +11,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/structs"
)
// MigrateOptions is equal to base.MigrateOptions
@ -30,6 +31,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
var (
downloader base.Downloader
uploader = NewGiteaLocalUploader(doer, ownerName, opts.RepoName)
theFactory base.DownloaderFactory
)
for _, factory := range factories {
@ -40,6 +42,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
if err != nil {
return nil, err
}
theFactory = factory
break
}
}
@ -52,10 +55,14 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
opts.Comments = false
opts.Issues = false
opts.PullRequests = false
opts.GitServiceType = structs.PlainGitService
downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr)
log.Trace("Will migrate from git: %s", opts.CloneAddr)
} else if opts.GitServiceType == structs.NotMigrated {
opts.GitServiceType = theFactory.GitServiceType()
}
uploader.gitServiceType = opts.GitServiceType
if err := migrateRepository(downloader, uploader, opts); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)

59
modules/migrations/update.go Normal file
View File

@ -0,0 +1,59 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"strconv"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/structs"
)
// UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID
func UpdateMigrationPosterID() {
for _, gitService := range structs.SupportedFullGitService {
if err := updateMigrationPosterIDByGitService(gitService); err != nil {
log.Error("updateMigrationPosterIDByGitService failed: %v", err)
}
}
}
func updateMigrationPosterIDByGitService(tp structs.GitServiceType) error {
provider := tp.Name()
if len(provider) == 0 {
return nil
}
const batchSize = 100
var start int
for {
users, err := models.FindExternalUsersByProvider(models.FindExternalUserOptions{
Provider: provider,
Start: start,
Limit: batchSize,
})
if err != nil {
return err
}
for _, user := range users {
externalUserID, err := strconv.ParseInt(user.ExternalID, 10, 64)
if err != nil {
log.Warn("Parse externalUser %#v 's userID failed: %v", user, err)
continue
}
if err := models.UpdateMigrationsByType(tp, externalUserID, user.UserID); err != nil {
log.Error("UpdateMigrationsByType type %s external user id %v to local user id %v failed: %v", tp.Name(), user.ExternalID, user.UserID, err)
}
}
if len(users) < batchSize {
break
}
start += len(users)
}
return nil
}