forgejo/services/migrations/gitea_uploader_test.go
Gergely Nagy ade7304eea
migrations: Map non-existant external users to Ghost
When performing migrations, and need to remap external users to local
ones, when no local mapping is possible, map the external user to Ghost,
rather than the user who initiated the migration.

Mapping the external user to the migration initiator has the potential
of breaking assumptions elsewhere, like only having one review per pull
request per user. Mapping these migrated, locally unavailable users to
Ghost makes sure these - often hidden - assumptions do not break.

Fixes .

Signed-off-by: Gergely Nagy <forgejo@gergo.csillger.hu>
2024-05-28 12:40:11 +02:00

518 lines
15 KiB
Go

// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"testing"
"time"
"code.gitea.io/gitea/models/db"
issues_model "code.gitea.io/gitea/models/issues"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/test"
"github.com/stretchr/testify/assert"
)
func TestGiteaUploadRepo(t *testing.T) {
// FIXME: Since no accesskey or user/password will trigger rate limit of github, just skip
t.Skip()
unittest.PrepareTestEnv(t)
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
var (
ctx = context.Background()
downloader = NewGithubDownloaderV3(ctx, "https://github.com", "", "", "", "go-xorm", "builder")
repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05")
uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
)
err := migrateRepository(db.DefaultContext, user, downloader, uploader, base.MigrateOptions{
CloneAddr: "https://github.com/go-xorm/builder",
RepoName: repoName,
AuthUsername: "",
Wiki: true,
Issues: true,
Milestones: true,
Labels: true,
Releases: true,
Comments: true,
PullRequests: true,
Private: true,
Mirror: false,
}, nil)
assert.NoError(t, err)
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerID: user.ID, Name: repoName})
assert.True(t, repo.HasWiki())
assert.EqualValues(t, repo_model.RepositoryReady, repo.Status)
milestones, err := db.Find[issues_model.Milestone](db.DefaultContext, issues_model.FindMilestoneOptions{
RepoID: repo.ID,
IsClosed: optional.Some(false),
})
assert.NoError(t, err)
assert.Len(t, milestones, 1)
milestones, err = db.Find[issues_model.Milestone](db.DefaultContext, issues_model.FindMilestoneOptions{
RepoID: repo.ID,
IsClosed: optional.Some(true),
})
assert.NoError(t, err)
assert.Empty(t, milestones)
labels, err := issues_model.GetLabelsByRepoID(ctx, repo.ID, "", db.ListOptions{})
assert.NoError(t, err)
assert.Len(t, labels, 12)
releases, err := db.Find[repo_model.Release](db.DefaultContext, repo_model.FindReleasesOptions{
ListOptions: db.ListOptions{
PageSize: 10,
Page: 0,
},
IncludeTags: true,
RepoID: repo.ID,
})
assert.NoError(t, err)
assert.Len(t, releases, 8)
releases, err = db.Find[repo_model.Release](db.DefaultContext, repo_model.FindReleasesOptions{
ListOptions: db.ListOptions{
PageSize: 10,
Page: 0,
},
IncludeTags: false,
RepoID: repo.ID,
})
assert.NoError(t, err)
assert.Len(t, releases, 1)
issues, err := issues_model.Issues(db.DefaultContext, &issues_model.IssuesOptions{
RepoIDs: []int64{repo.ID},
IsPull: optional.Some(false),
SortType: "oldest",
})
assert.NoError(t, err)
assert.Len(t, issues, 15)
assert.NoError(t, issues[0].LoadDiscussComments(db.DefaultContext))
assert.Empty(t, issues[0].Comments)
pulls, _, err := issues_model.PullRequests(db.DefaultContext, repo.ID, &issues_model.PullRequestsOptions{
SortType: "oldest",
})
assert.NoError(t, err)
assert.Len(t, pulls, 30)
assert.NoError(t, pulls[0].LoadIssue(db.DefaultContext))
assert.NoError(t, pulls[0].Issue.LoadDiscussComments(db.DefaultContext))
assert.Len(t, pulls[0].Issue.Comments, 2)
}
func TestGiteaUploadRemapLocalUser(t *testing.T) {
unittest.PrepareTestEnv(t)
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
repoName := "migrated"
uploader := NewGiteaLocalUploader(context.Background(), doer, doer.Name, repoName)
// call remapLocalUser
uploader.sameApp = true
externalID := int64(1234567)
externalName := "username"
source := base.Release{
PublisherID: externalID,
PublisherName: externalName,
}
//
// The externalID does not match any existing user, everything
// belongs to the Ghost user
//
target := repo_model.Release{}
uploader.userMap = make(map[int64]int64)
err := uploader.remapUser(&source, &target)
assert.NoError(t, err)
assert.EqualValues(t, user_model.GhostUserID, target.GetUserID())
//
// The externalID matches a known user but the name does not match,
// everything belongs to the Ghost user
//
source.PublisherID = user.ID
target = repo_model.Release{}
uploader.userMap = make(map[int64]int64)
err = uploader.remapUser(&source, &target)
assert.NoError(t, err)
assert.EqualValues(t, user_model.GhostUserID, target.GetUserID())
//
// The externalID and externalName match an existing user, everything
// belongs to the existing user
//
source.PublisherName = user.Name
target = repo_model.Release{}
uploader.userMap = make(map[int64]int64)
err = uploader.remapUser(&source, &target)
assert.NoError(t, err)
assert.EqualValues(t, user.ID, target.GetUserID())
}
func TestGiteaUploadRemapExternalUser(t *testing.T) {
unittest.PrepareTestEnv(t)
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
repoName := "migrated"
uploader := NewGiteaLocalUploader(context.Background(), doer, doer.Name, repoName)
uploader.gitServiceType = structs.GiteaService
// call remapExternalUser
uploader.sameApp = false
externalID := int64(1234567)
externalName := "username"
source := base.Release{
PublisherID: externalID,
PublisherName: externalName,
}
//
// When there is no user linked to the external ID, the migrated data is authored
// by the Ghost user
//
uploader.userMap = make(map[int64]int64)
target := repo_model.Release{}
err := uploader.remapUser(&source, &target)
assert.NoError(t, err)
assert.EqualValues(t, user_model.GhostUserID, target.GetUserID())
//
// Link the external ID to an existing user
//
linkedUser := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
externalLoginUser := &user_model.ExternalLoginUser{
ExternalID: strconv.FormatInt(externalID, 10),
UserID: linkedUser.ID,
LoginSourceID: 0,
Provider: structs.GiteaService.Name(),
}
err = user_model.LinkExternalToUser(db.DefaultContext, linkedUser, externalLoginUser)
assert.NoError(t, err)
//
// When a user is linked to the external ID, it becomes the author of
// the migrated data
//
uploader.userMap = make(map[int64]int64)
target = repo_model.Release{}
err = uploader.remapUser(&source, &target)
assert.NoError(t, err)
assert.EqualValues(t, linkedUser.ID, target.GetUserID())
}
func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) {
unittest.PrepareTestEnv(t)
//
// fromRepo master
//
fromRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
baseRef := "master"
assert.NoError(t, git.InitRepository(git.DefaultContext, fromRepo.RepoPath(), false, fromRepo.ObjectFormatName))
err := git.NewCommand(git.DefaultContext, "symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseRef).Run(&git.RunOpts{Dir: fromRepo.RepoPath()})
assert.NoError(t, err)
assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# Testing Repository\n\nOriginally created in: %s", fromRepo.RepoPath())), 0o644))
assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true))
signature := git.Signature{
Email: "test@example.com",
Name: "test",
When: time.Now(),
}
assert.NoError(t, git.CommitChanges(fromRepo.RepoPath(), git.CommitChangesOptions{
Committer: &signature,
Author: &signature,
Message: "Initial Commit",
}))
fromGitRepo, err := gitrepo.OpenRepository(git.DefaultContext, fromRepo)
assert.NoError(t, err)
defer fromGitRepo.Close()
baseSHA, err := fromGitRepo.GetBranchCommitID(baseRef)
assert.NoError(t, err)
//
// fromRepo branch1
//
headRef := "branch1"
_, _, err = git.NewCommand(git.DefaultContext, "checkout", "-b").AddDynamicArguments(headRef).RunStdString(&git.RunOpts{Dir: fromRepo.RepoPath()})
assert.NoError(t, err)
assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte("SOMETHING"), 0o644))
assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true))
signature.When = time.Now()
assert.NoError(t, git.CommitChanges(fromRepo.RepoPath(), git.CommitChangesOptions{
Committer: &signature,
Author: &signature,
Message: "Pull request",
}))
assert.NoError(t, err)
headSHA, err := fromGitRepo.GetBranchCommitID(headRef)
assert.NoError(t, err)
fromRepoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: fromRepo.OwnerID})
//
// forkRepo branch2
//
forkHeadRef := "branch2"
forkRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 8})
assert.NoError(t, git.CloneWithArgs(git.DefaultContext, nil, fromRepo.RepoPath(), forkRepo.RepoPath(), git.CloneRepoOptions{
Branch: headRef,
}))
_, _, err = git.NewCommand(git.DefaultContext, "checkout", "-b").AddDynamicArguments(forkHeadRef).RunStdString(&git.RunOpts{Dir: forkRepo.RepoPath()})
assert.NoError(t, err)
assert.NoError(t, os.WriteFile(filepath.Join(forkRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# branch2 %s", forkRepo.RepoPath())), 0o644))
assert.NoError(t, git.AddChanges(forkRepo.RepoPath(), true))
assert.NoError(t, git.CommitChanges(forkRepo.RepoPath(), git.CommitChangesOptions{
Committer: &signature,
Author: &signature,
Message: "branch2 commit",
}))
forkGitRepo, err := gitrepo.OpenRepository(git.DefaultContext, forkRepo)
assert.NoError(t, err)
defer forkGitRepo.Close()
forkHeadSHA, err := forkGitRepo.GetBranchCommitID(forkHeadRef)
assert.NoError(t, err)
toRepoName := "migrated"
uploader := NewGiteaLocalUploader(context.Background(), fromRepoOwner, fromRepoOwner.Name, toRepoName)
uploader.gitServiceType = structs.GiteaService
assert.NoError(t, uploader.CreateRepo(&base.Repository{
Description: "description",
OriginalURL: fromRepo.RepoPath(),
CloneURL: fromRepo.RepoPath(),
IsPrivate: false,
IsMirror: true,
}, base.MigrateOptions{
GitServiceType: structs.GiteaService,
Private: false,
Mirror: true,
}))
for _, testCase := range []struct {
name string
head string
logFilter []string
logFiltered []bool
pr base.PullRequest
}{
{
name: "fork, good Head.SHA",
head: fmt.Sprintf("%s/%s", forkRepo.OwnerName, forkHeadRef),
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: forkRepo.RepoPath(),
Ref: forkHeadRef,
SHA: forkHeadSHA,
RepoName: forkRepo.Name,
OwnerName: forkRepo.OwnerName,
},
},
},
{
name: "fork, invalid Head.Ref",
head: "unknown repository",
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: forkRepo.RepoPath(),
Ref: "INVALID",
SHA: forkHeadSHA,
RepoName: forkRepo.Name,
OwnerName: forkRepo.OwnerName,
},
},
logFilter: []string{"Fetch branch from"},
logFiltered: []bool{true},
},
{
name: "invalid fork CloneURL",
head: "unknown repository",
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: "UNLIKELY",
Ref: forkHeadRef,
SHA: forkHeadSHA,
RepoName: forkRepo.Name,
OwnerName: "WRONG",
},
},
logFilter: []string{"AddRemote"},
logFiltered: []bool{true},
},
{
name: "no fork, good Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: headSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
},
{
name: "no fork, empty Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: "",
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
logFilter: []string{"Empty reference", "Cannot remove local head"},
logFiltered: []bool{true, false},
},
{
name: "no fork, invalid Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: "brokenSHA",
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
logFilter: []string{"Deprecated local head"},
logFiltered: []bool{true},
},
{
name: "no fork, not found Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: "2697b352310fcd01cbd1f3dbd43b894080027f68",
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
logFilter: []string{"Deprecated local head", "Cannot remove local head"},
logFiltered: []bool{true, false},
},
} {
t.Run(testCase.name, func(t *testing.T) {
stopMark := fmt.Sprintf(">>>>>>>>>>>>>STOP: %s<<<<<<<<<<<<<<<", testCase.name)
logChecker, cleanup := test.NewLogChecker(log.DEFAULT, log.INFO)
logChecker.Filter(testCase.logFilter...).StopMark(stopMark)
defer cleanup()
testCase.pr.EnsuredSafe = true
head, err := uploader.updateGitForPullRequest(&testCase.pr)
assert.NoError(t, err)
assert.EqualValues(t, testCase.head, head)
log.Info(stopMark)
logFiltered, logStopped := logChecker.Check(5 * time.Second)
assert.True(t, logStopped)
if len(testCase.logFilter) > 0 {
assert.EqualValues(t, testCase.logFiltered, logFiltered, "for log message filters: %v", testCase.logFilter)
}
})
}
}