Infrastructure/forgejo
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-12-22 23:33:15 -05:00
Code Issues Activity

migrations: add test for importing pull requests in gitea uploader (#18752)

Browse source 
* logs: add the buffer logger to inspect logs during testing

Signed-off-by: Loïc Dachary <loic@dachary.org>

* migrations: add test for importing pull requests in gitea uploader

Signed-off-by: Loïc Dachary <loic@dachary.org>

* for each git.OpenRepositoryCtx, call Close

* Content is expected to return the content of the log

* test for errors before defer

Co-authored-by: Loïc Dachary <loic@dachary.org>
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
singuliere 2022-02-25 10:20:50 +01:00 committed by GitHub
parent e4ef61ee0f
commit 49cab2b01f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 512 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
integrations/testlogger.go
View file

@ -181,6 +181,11 @@ func (log *TestLogger) Init(config string) error {
return nil
}
// Content returns the content accumulated in the content provider
func (log *TestLogger) Content() (string, error) {
return "", fmt.Errorf("not supported")
}
// Flush when log should be flushed
func (log *TestLogger) Flush() {
}

5
models/migrations/testlogger_test.go
View file

@ -166,6 +166,11 @@ func (log *TestLogger) Init(config string) error {
return nil
}
// Content returns the content accumulated in the content provider
func (log *TestLogger) Content() (string, error) {
return "", fmt.Errorf("not supported")
}
// Flush when log should be flushed
func (log *TestLogger) Flush() {
}

72
modules/log/buffer.go Normal file
View file

@ -0,0 +1,72 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package log
import (
"bytes"
"sync"
)
type bufferWriteCloser struct {
mu sync.Mutex
buffer bytes.Buffer
}
func (b *bufferWriteCloser) Write(p []byte) (int, error) {
b.mu.Lock()
defer b.mu.Unlock()
return b.buffer.Write(p)
}
func (b *bufferWriteCloser) Close() error {
return nil
}
func (b *bufferWriteCloser) String() string {
b.mu.Lock()
defer b.mu.Unlock()
return b.buffer.String()
}
// BufferLogger implements LoggerProvider and writes messages in a buffer.
type BufferLogger struct {
WriterLogger
}
// NewBufferLogger create BufferLogger returning as LoggerProvider.
func NewBufferLogger() LoggerProvider {
log := &BufferLogger{}
log.NewWriterLogger(&bufferWriteCloser{})
return log
}
// Init inits connection writer
func (log *BufferLogger) Init(string) error {
log.NewWriterLogger(log.out)
return nil
}
// Content returns the content accumulated in the content provider
func (log *BufferLogger) Content() (string, error) {
return log.out.(*bufferWriteCloser).String(), nil
}
// Flush when log should be flushed
func (log *BufferLogger) Flush() {
}
// ReleaseReopen does nothing
func (log *BufferLogger) ReleaseReopen() error {
return nil
}
// GetName returns the default name for this implementation
func (log *BufferLogger) GetName() string {
return "buffer"
}
func init() {
Register("buffer", NewBufferLogger)
}

64
modules/log/buffer_test.go Normal file
View file

@ -0,0 +1,64 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package log
import (
"fmt"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestBufferLogger(t *testing.T) {
logger := NewBufferLogger()
bufferLogger := logger.(*BufferLogger)
assert.NotNil(t, bufferLogger)
err := logger.Init("")
assert.NoError(t, err)
location, _ := time.LoadLocation("EST")
date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location)
msg := "TEST MSG"
event := Event{
level: INFO,
msg: msg,
caller: "CALLER",
filename: "FULL/FILENAME",
line: 1,
time: date,
}
logger.LogEvent(&event)
content, err := bufferLogger.Content()
assert.NoError(t, err)
assert.Contains(t, content, msg)
logger.Close()
}
func TestBufferLoggerContent(t *testing.T) {
level := INFO
logger := NewLogger(0, "console", "console", fmt.Sprintf(`{"level":"%s"}`, level.String()))
logger.SetLogger("buffer", "buffer", "{}")
defer logger.DelLogger("buffer")
msg := "A UNIQUE MESSAGE"
Error(msg)
found := false
for i := 0; i < 30000; i++ {
content, err := logger.GetLoggerProviderContent("buffer")
assert.NoError(t, err)
if strings.Contains(content, msg) {
found = true
break
}
time.Sleep(1 * time.Millisecond)
}
assert.True(t, found)
}

5
modules/log/conn.go
View file

@ -119,6 +119,11 @@ func (log *ConnLogger) Init(jsonconfig string) error {
return nil
}
// Content returns the content accumulated in the content provider
func (log *ConnLogger) Content() (string, error) {
return "", fmt.Errorf("not supported")
}
// Flush does nothing for this implementation
func (log *ConnLogger) Flush() {
}

5
modules/log/console.go
View file

@ -66,6 +66,11 @@ func (log *ConsoleLogger) Init(config string) error {
return nil
}
// Content returns the content accumulated in the content provider
func (log *ConsoleLogger) Content() (string, error) {
return "", fmt.Errorf("not supported")
}
// Flush when log should be flushed
func (log *ConsoleLogger) Flush() {
}

6
modules/log/event.go
View file

@ -216,6 +216,12 @@ func (m *MultiChannelledLog) GetEventLogger(name string) EventLogger {
return m.loggers[name]
}
// GetEventProvider returns a sub logger provider content from this MultiChannelledLog
func (m *MultiChannelledLog) GetLoggerProviderContent(name string) (string, error) {
channelledLogger := m.GetEventLogger(name).(*ChannelledLog)
return channelledLogger.loggerProvider.Content()
}
// GetEventLoggerNames returns a list of names
func (m *MultiChannelledLog) GetEventLoggerNames() []string {
m.rwmutex.RLock()

9
modules/log/file.go
View file

@ -243,6 +243,15 @@ func (log *FileLogger) deleteOldLog() {
})
}
// Content returns the content accumulated in the content provider
func (log *FileLogger) Content() (string, error) {
b, err := os.ReadFile(log.Filename)
if err != nil {
return "", err
}
return string(b), nil
}
// Flush flush file logger.
// there are no buffering messages in file logger in memory.
// flush file means sync file from disk.

1
modules/log/provider.go
View file

@ -7,6 +7,7 @@ package log
// LoggerProvider represents behaviors of a logger provider.
type LoggerProvider interface {
Init(config string) error
Content() (string, error)
EventLogger
}

5
modules/log/smtp.go
View file

@ -95,6 +95,11 @@ func (log *SMTPLogger) sendMail(p []byte) (int, error) {
)
}
// Content returns the content accumulated in the content provider
func (log *SMTPLogger) Content() (string, error) {
return "", fmt.Errorf("not supported")
}
// Flush when log should be flushed
func (log *SMTPLogger) Flush() {
}

49
services/migrations/gitea_uploader.go
View file

@ -484,19 +484,9 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error
return nil
}
func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) {
var labels []*models.Label
for _, label := range pr.Labels {
lb, ok := g.labels[label.Name]
if ok {
labels = append(labels, lb)
}
}
milestoneID := g.milestones[pr.Milestone]
func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) {
// download patch file
err := func() error {
err = func() error {
if pr.PatchURL == "" {
return nil
}
@ -519,25 +509,25 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
return err
}()
if err != nil {
return nil, err
return "", err
}
// set head information
pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
return nil, err
return "", err
}
p, err := os.Create(filepath.Join(pullHead, "head"))
if err != nil {
return nil, err
return "", err
}
_, err = p.WriteString(pr.Head.SHA)
p.Close()
if err != nil {
return nil, err
return "", err
}
head := "unknown repository"
head = "unknown repository"
if pr.IsForkPullRequest() && pr.State != "closed" {
if pr.Head.OwnerName != "" {
remote := pr.Head.OwnerName
@ -560,16 +550,16 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
} else {
headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
return nil, err
return "", err
}
b, err := os.Create(headBranch)
if err != nil {
return nil, err
return "", err
}
_, err = b.WriteString(pr.Head.SHA)
b.Close()
if err != nil {
return nil, err
return "", err
}
head = pr.Head.OwnerName + "/" + pr.Head.Ref
}
@ -595,6 +585,25 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
}
}
return head, nil
}
func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) {
var labels []*models.Label
for _, label := range pr.Labels {
lb, ok := g.labels[label.Name]
if ok {
labels = append(labels, lb)
}
}
milestoneID := g.milestones[pr.Milestone]
head, err := g.updateGitForPullRequest(pr)
if err != nil {
return nil, fmt.Errorf("updateGitForPullRequest: %w", err)
}
if pr.Created.IsZero() {
if pr.Closed != nil {
pr.Created = *pr.Closed

306
services/migrations/gitea_uploader_test.go
View file

@ -7,7 +7,11 @@ package migrations
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
@ -16,7 +20,9 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
@ -217,3 +223,303 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) {
assert.NoError(t, err)
assert.EqualValues(t, linkedUser.ID, target.GetUserID())
}
func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) {
unittest.PrepareTestEnv(t)
//
// fromRepo master
//
fromRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}).(*repo_model.Repository)
baseRef := "master"
assert.NoError(t, git.InitRepository(git.DefaultContext, fromRepo.RepoPath(), false))
_, err := git.NewCommand(git.DefaultContext, "symbolic-ref", "HEAD", git.BranchPrefix+baseRef).RunInDir(fromRepo.RepoPath())
assert.NoError(t, err)
assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# Testing Repository\n\nOriginally created in: %s", fromRepo.RepoPath())), 0o644))
assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true))
signature := git.Signature{
Email: "test@example.com",
Name: "test",
When: time.Now(),
}
assert.NoError(t, git.CommitChanges(fromRepo.RepoPath(), git.CommitChangesOptions{
Committer: &signature,
Author: &signature,
Message: "Initial Commit",
}))
fromGitRepo, err := git.OpenRepositoryCtx(git.DefaultContext, fromRepo.RepoPath())
assert.NoError(t, err)
defer fromGitRepo.Close()
baseSHA, err := fromGitRepo.GetBranchCommitID(baseRef)
assert.NoError(t, err)
//
// fromRepo branch1
//
headRef := "branch1"
_, err = git.NewCommand(git.DefaultContext, "checkout", "-b", headRef).RunInDir(fromRepo.RepoPath())
assert.NoError(t, err)
assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte("SOMETHING"), 0o644))
assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true))
signature.When = time.Now()
assert.NoError(t, git.CommitChanges(fromRepo.RepoPath(), git.CommitChangesOptions{
Committer: &signature,
Author: &signature,
Message: "Pull request",
}))
assert.NoError(t, err)
headSHA, err := fromGitRepo.GetBranchCommitID(headRef)
assert.NoError(t, err)
fromRepoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: fromRepo.OwnerID}).(*user_model.User)
//
// forkRepo branch2
//
forkHeadRef := "branch2"
forkRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 8}).(*repo_model.Repository)
assert.NoError(t, git.CloneWithArgs(git.DefaultContext, fromRepo.RepoPath(), forkRepo.RepoPath(), []string{}, git.CloneRepoOptions{
Branch: headRef,
}))
_, err = git.NewCommand(git.DefaultContext, "checkout", "-b", forkHeadRef).RunInDir(forkRepo.RepoPath())
assert.NoError(t, err)
assert.NoError(t, os.WriteFile(filepath.Join(forkRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# branch2 %s", forkRepo.RepoPath())), 0o644))
assert.NoError(t, git.AddChanges(forkRepo.RepoPath(), true))
assert.NoError(t, git.CommitChanges(forkRepo.RepoPath(), git.CommitChangesOptions{
Committer: &signature,
Author: &signature,
Message: "branch2 commit",
}))
forkGitRepo, err := git.OpenRepositoryCtx(git.DefaultContext, forkRepo.RepoPath())
assert.NoError(t, err)
defer forkGitRepo.Close()
forkHeadSHA, err := forkGitRepo.GetBranchCommitID(forkHeadRef)
assert.NoError(t, err)
toRepoName := "migrated"
uploader := NewGiteaLocalUploader(context.Background(), fromRepoOwner, fromRepoOwner.Name, toRepoName)
uploader.gitServiceType = structs.GiteaService
assert.NoError(t, uploader.CreateRepo(&base.Repository{
Description: "description",
OriginalURL: fromRepo.RepoPath(),
CloneURL: fromRepo.RepoPath(),
IsPrivate: false,
IsMirror: true,
}, base.MigrateOptions{
GitServiceType: structs.GiteaService,
Private: false,
Mirror: true,
}))
for _, testCase := range []struct {
name string
head string
assertContent func(t *testing.T, content string)
pr base.PullRequest
}{
{
name: "fork, good Head.SHA",
head: fmt.Sprintf("%s/%s", forkRepo.OwnerName, forkHeadRef),
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: forkRepo.RepoPath(),
Ref: forkHeadRef,
SHA: forkHeadSHA,
RepoName: forkRepo.Name,
OwnerName: forkRepo.OwnerName,
},
},
},
{
name: "fork, invalid Head.Ref",
head: "unknown repository",
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: forkRepo.RepoPath(),
Ref: "INVALID",
SHA: forkHeadSHA,
RepoName: forkRepo.Name,
OwnerName: forkRepo.OwnerName,
},
},
assertContent: func(t *testing.T, content string) {
assert.Contains(t, content, "Fetch branch from")
},
},
{
name: "invalid fork CloneURL",
head: "unknown repository",
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: "UNLIKELY",
Ref: forkHeadRef,
SHA: forkHeadSHA,
RepoName: forkRepo.Name,
OwnerName: "WRONG",
},
},
assertContent: func(t *testing.T, content string) {
assert.Contains(t, content, "AddRemote failed")
},
},
{
name: "no fork, good Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: headSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
},
{
name: "no fork, empty Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: "",
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
assertContent: func(t *testing.T, content string) {
assert.Contains(t, content, "Empty reference, removing")
assert.NotContains(t, content, "Cannot remove local head")
},
},
{
name: "no fork, invalid Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: "brokenSHA",
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
assertContent: func(t *testing.T, content string) {
assert.Contains(t, content, "Deprecated local head")
assert.Contains(t, content, "Cannot remove local head")
},
},
{
name: "no fork, not found Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: "2697b352310fcd01cbd1f3dbd43b894080027f68",
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
assertContent: func(t *testing.T, content string) {
assert.Contains(t, content, "Deprecated local head")
assert.NotContains(t, content, "Cannot remove local head")
},
},
} {
t.Run(testCase.name, func(t *testing.T) {
logger, ok := log.NamedLoggers.Load(log.DEFAULT)
assert.True(t, ok)
logger.SetLogger("buffer", "buffer", "{}")
defer logger.DelLogger("buffer")
head, err := uploader.updateGitForPullRequest(&testCase.pr)
assert.NoError(t, err)
assert.EqualValues(t, testCase.head, head)
if testCase.assertContent != nil {
fence := fmt.Sprintf(">>>>>>>>>>>>>FENCE %s<<<<<<<<<<<<<<<", testCase.name)
log.Error(fence)
var content string
for i := 0; i < 5000; i++ {
content, err = logger.GetLoggerProviderContent("buffer")
assert.NoError(t, err)
if strings.Contains(content, fence) {
break
}
time.Sleep(1 * time.Millisecond)
}
testCase.assertContent(t, content)
}
})
}
}