From bf934c96c92d643678ac7a18697b6563bc9d20a5 Mon Sep 17 00:00:00 2001 From: "Sebastian T. T." <109338575+Sebastian-T-T@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:20:04 +0100 Subject: [PATCH 01/11] Add standard-compliant route to serve outdated R packages (#32783) The R package repository currently does not have support for older versions of packages which should be stored in a separate /Archive router. This PR remedies that by adding a new path router. I am a member of a group that loves using Gitea and this bug has been annoying us for a long time. Hope it can be merged in time for Gitea 1.23.0. Any feedback much appreciated. Fixes #32782 (cherry picked from commit 874b8484aa9f7e10172fd1a8a7c768e70b36c475) --- routers/api/packages/api.go | 1 + tests/integration/api_packages_cran_test.go | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 781bfc7f90..e216a0c02b 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -337,6 +337,7 @@ func CommonRoutes() *web.Route { r.Get("/PACKAGES", cran.EnumerateSourcePackages) r.Get("/PACKAGES{format}", cran.EnumerateSourcePackages) r.Get("/{filename}", cran.DownloadSourcePackageFile) + r.Get("/Archive/{packagename}/{filename}", cran.DownloadSourcePackageFile) }) r.Put("", reqPackageAccess(perm.AccessModeWrite), enforcePackagesQuota(), cran.UploadSourcePackageFile) }) diff --git a/tests/integration/api_packages_cran_test.go b/tests/integration/api_packages_cran_test.go index 31864d1ab7..d64b592327 100644 --- a/tests/integration/api_packages_cran_test.go +++ b/tests/integration/api_packages_cran_test.go @@ -116,6 +116,14 @@ func TestPackageCran(t *testing.T) { MakeRequest(t, req, http.StatusOK) }) + t.Run("DownloadArchived", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/src/contrib/Archive/%s/%s_%s.tar.gz", url, packageName, packageName, packageVersion)). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusOK) + }) + t.Run("Enumerate", func(t *testing.T) { defer tests.PrintCurrentTest(t)() From c3d37894aad0e60341b24012af4edb725a70b2e5 Mon Sep 17 00:00:00 2001 From: RiceChuan Date: Thu, 12 Dec 2024 12:26:11 +0800 Subject: [PATCH 02/11] chore: use errors.New to replace fmt.Errorf with no parameters (#32800) use errors.New to replace fmt.Errorf with no parameters Signed-off-by: RiceChuan (cherry picked from commit dfd75944992fc6508ec891b4c29715c23e59e4ed) --- models/db/context_committer_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/db/context_committer_test.go b/models/db/context_committer_test.go index 38e91f22ed..849c5dea41 100644 --- a/models/db/context_committer_test.go +++ b/models/db/context_committer_test.go @@ -4,7 +4,7 @@ package db // it's not db_test, because this file is for testing the private type halfCommitter import ( - "fmt" + "errors" "testing" "github.com/stretchr/testify/assert" @@ -80,7 +80,7 @@ func Test_halfCommitter(t *testing.T) { testWithCommitter(mockCommitter, func(committer Committer) error { defer committer.Close() if true { - return fmt.Errorf("error") + return errors.New("error") } return committer.Commit() }) @@ -94,7 +94,7 @@ func Test_halfCommitter(t *testing.T) { testWithCommitter(mockCommitter, func(committer Committer) error { committer.Close() committer.Commit() - return fmt.Errorf("error") + return errors.New("error") }) mockCommitter.Assert(t) From b0d6a7f07bff836190a8e87fe5645d5557893e32 Mon Sep 17 00:00:00 2001 From: Kemal Zebari <60799661+kemzeb@users.noreply.github.com> Date: Wed, 11 Dec 2024 21:02:35 -0800 Subject: [PATCH 03/11] Implement update branch API (#32433) Resolves #22526. Builds upon #23061. --------- Co-authored-by: sillyguodong <33891828+sillyguodong@users.noreply.github.com> Co-authored-by: wxiaoguang (cherry picked from commit 01b1896bf5eacfd7f4f64d9ebb0ad165e3e60a5c) Conflicts: routers/api/v1/api.go context conflict + s/PathParam/Params/ templates/swagger/v1_json.tmpl make generate-swagger --- modules/structs/repo.go | 10 ++++ routers/api/v1/api.go | 1 + routers/api/v1/repo/branch.go | 71 +++++++++++++++++++++++++++ routers/api/v1/swagger/options.go | 2 + templates/swagger/v1_json.tmpl | 73 ++++++++++++++++++++++++++++ tests/integration/api_branch_test.go | 32 ++++++++++++ 6 files changed, 189 insertions(+) diff --git a/modules/structs/repo.go b/modules/structs/repo.go index f2fe9c7ac3..36190fe36b 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -290,6 +290,16 @@ type CreateBranchRepoOption struct { OldRefName string `json:"old_ref_name" binding:"GitRefName;MaxSize(100)"` } +// UpdateBranchRepoOption options when updating a branch in a repository +// swagger:model +type UpdateBranchRepoOption struct { + // New branch name + // + // required: true + // unique: true + Name string `json:"name" binding:"Required;GitRefName;MaxSize(100)"` +} + // TransferRepoOption options when transfer a repository's ownership // swagger:model type TransferRepoOption struct { diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c028a5a32d..4928c9ff58 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1153,6 +1153,7 @@ func Routes() *web.Route { m.Get("/*", repo.GetBranch) m.Delete("/*", reqToken(), reqRepoWriter(unit.TypeCode), mustNotBeArchived, repo.DeleteBranch) m.Post("", reqToken(), reqRepoWriter(unit.TypeCode), mustNotBeArchived, bind(api.CreateBranchRepoOption{}), context.EnforceQuotaAPI(quota_model.LimitSubjectSizeGitAll, context.QuotaTargetRepo), repo.CreateBranch) + m.Patch("/*", reqToken(), reqRepoWriter(unit.TypeCode), mustNotBeArchived, bind(api.UpdateBranchRepoOption{}), repo.UpdateBranch) }, context.ReferencesGitRepo(), reqRepoReader(unit.TypeCode)) m.Group("/branch_protections", func() { m.Get("", repo.ListBranchProtections) diff --git a/routers/api/v1/repo/branch.go b/routers/api/v1/repo/branch.go index e3e6efa781..3ca97f7770 100644 --- a/routers/api/v1/repo/branch.go +++ b/routers/api/v1/repo/branch.go @@ -393,6 +393,77 @@ func ListBranches(ctx *context.APIContext) { ctx.JSON(http.StatusOK, apiBranches) } +// UpdateBranch updates a repository's branch. +func UpdateBranch(ctx *context.APIContext) { + // swagger:operation PATCH /repos/{owner}/{repo}/branches/{branch} repository repoUpdateBranch + // --- + // summary: Update a branch + // consumes: + // - application/json + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: branch + // in: path + // description: name of the branch + // type: string + // required: true + // - name: body + // in: body + // schema: + // "$ref": "#/definitions/UpdateBranchRepoOption" + // responses: + // "204": + // "$ref": "#/responses/empty" + // "403": + // "$ref": "#/responses/forbidden" + // "404": + // "$ref": "#/responses/notFound" + // "422": + // "$ref": "#/responses/validationError" + + opt := web.GetForm(ctx).(*api.UpdateBranchRepoOption) + + oldName := ctx.Params("*") + repo := ctx.Repo.Repository + + if repo.IsEmpty { + ctx.Error(http.StatusNotFound, "", "Git Repository is empty.") + return + } + + if repo.IsMirror { + ctx.Error(http.StatusForbidden, "", "Git Repository is a mirror.") + return + } + + msg, err := repo_service.RenameBranch(ctx, repo, ctx.Doer, ctx.Repo.GitRepo, oldName, opt.Name) + if err != nil { + ctx.Error(http.StatusInternalServerError, "RenameBranch", err) + return + } + if msg == "target_exist" { + ctx.Error(http.StatusUnprocessableEntity, "", "Cannot rename a branch using the same name or rename to a branch that already exists.") + return + } + if msg == "from_not_exist" { + ctx.Error(http.StatusNotFound, "", "Branch doesn't exist.") + return + } + + ctx.Status(http.StatusNoContent) +} + // GetBranchProtection gets a branch protection func GetBranchProtection(ctx *context.APIContext) { // swagger:operation GET /repos/{owner}/{repo}/branch_protections/{name} repository repoGetBranchProtection diff --git a/routers/api/v1/swagger/options.go b/routers/api/v1/swagger/options.go index 1dccf92d82..432e42d4e7 100644 --- a/routers/api/v1/swagger/options.go +++ b/routers/api/v1/swagger/options.go @@ -101,6 +101,8 @@ type swaggerParameterBodies struct { // in:body EditRepoOption api.EditRepoOption // in:body + UpdateBranchRepoOption api.UpdateBranchRepoOption + // in:body TransferRepoOption api.TransferRepoOption // in:body CreateForkOption api.CreateForkOption diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index b5fd179d79..3a7d12664c 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -5806,6 +5806,63 @@ "$ref": "#/responses/repoArchivedError" } } + }, + "patch": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "repository" + ], + "summary": "Update a branch", + "operationId": "repoUpdateBranch", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the branch", + "name": "branch", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "schema": { + "$ref": "#/definitions/UpdateBranchRepoOption" + } + } + ], + "responses": { + "204": { + "$ref": "#/responses/empty" + }, + "403": { + "$ref": "#/responses/forbidden" + }, + "404": { + "$ref": "#/responses/notFound" + }, + "422": { + "$ref": "#/responses/validationError" + } + } } }, "/repos/{owner}/{repo}/collaborators": { @@ -27288,6 +27345,22 @@ }, "x-go-package": "code.gitea.io/gitea/modules/structs" }, + "UpdateBranchRepoOption": { + "description": "UpdateBranchRepoOption options when updating a branch in a repository", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "description": "New branch name", + "type": "string", + "uniqueItems": true, + "x-go-name": "Name" + } + }, + "x-go-package": "code.gitea.io/gitea/modules/structs" + }, "UpdateFileOptions": { "description": "UpdateFileOptions options for updating files\nNote: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used)", "type": "object", diff --git a/tests/integration/api_branch_test.go b/tests/integration/api_branch_test.go index 41b2c2efd1..aa22b15ea1 100644 --- a/tests/integration/api_branch_test.go +++ b/tests/integration/api_branch_test.go @@ -5,6 +5,7 @@ package integration import ( "net/http" + "net/http/httptest" "net/url" "testing" @@ -187,6 +188,37 @@ func testAPICreateBranch(t testing.TB, session *TestSession, user, repo, oldBran return resp.Result().StatusCode == status } +func TestAPIUpdateBranch(t *testing.T) { + onGiteaRun(t, func(t *testing.T, _ *url.URL) { + t.Run("UpdateBranchWithEmptyRepo", func(t *testing.T) { + testAPIUpdateBranch(t, "user10", "repo6", "master", "test", http.StatusNotFound) + }) + t.Run("UpdateBranchWithSameBranchNames", func(t *testing.T) { + resp := testAPIUpdateBranch(t, "user2", "repo1", "master", "master", http.StatusUnprocessableEntity) + assert.Contains(t, resp.Body.String(), "Cannot rename a branch using the same name or rename to a branch that already exists.") + }) + t.Run("UpdateBranchThatAlreadyExists", func(t *testing.T) { + resp := testAPIUpdateBranch(t, "user2", "repo1", "master", "branch2", http.StatusUnprocessableEntity) + assert.Contains(t, resp.Body.String(), "Cannot rename a branch using the same name or rename to a branch that already exists.") + }) + t.Run("UpdateBranchWithNonExistentBranch", func(t *testing.T) { + resp := testAPIUpdateBranch(t, "user2", "repo1", "i-dont-exist", "new-branch-name", http.StatusNotFound) + assert.Contains(t, resp.Body.String(), "Branch doesn't exist.") + }) + t.Run("RenameBranchNormalScenario", func(t *testing.T) { + testAPIUpdateBranch(t, "user2", "repo1", "branch2", "new-branch-name", http.StatusNoContent) + }) + }) +} + +func testAPIUpdateBranch(t *testing.T, ownerName, repoName, from, to string, expectedHTTPStatus int) *httptest.ResponseRecorder { + token := getUserToken(t, ownerName, auth_model.AccessTokenScopeWriteRepository) + req := NewRequestWithJSON(t, "PATCH", "api/v1/repos/"+ownerName+"/"+repoName+"/branches/"+from, &api.UpdateBranchRepoOption{ + Name: to, + }).AddTokenAuth(token) + return MakeRequest(t, req, expectedHTTPStatus) +} + func TestAPIBranchProtection(t *testing.T) { defer tests.PrepareTestEnv(t)() From 2b5c69c451a684b20119e2521dc23734c7869241 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 12 Dec 2024 11:28:23 -0800 Subject: [PATCH 04/11] Detect whether action view branch was deleted (#32764) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #32761 ![图片](https://github.com/user-attachments/assets/a5a7eef8-0fea-4242-b199-1b0b73d9bbdb) (cherry picked from commit 6370d2fb93a5ee897b82969ca30a9feb33667714) Conflicts: routers/web/repo/actions/actions.go routers/web/repo/actions/view.go trivial context conflicts --- models/actions/run.go | 1 + models/git/branch.go | 18 +++++++++++-- routers/web/repo/actions/actions.go | 34 ++++++++++++++++++++++++ routers/web/repo/actions/view.go | 18 +++++++++++-- services/repository/branch.go | 2 +- templates/repo/actions/runs_list.tmpl | 6 ++--- web_src/js/components/RepoActionView.vue | 3 ++- 7 files changed, 73 insertions(+), 9 deletions(-) diff --git a/models/actions/run.go b/models/actions/run.go index 06a1290d5d..c5512106b9 100644 --- a/models/actions/run.go +++ b/models/actions/run.go @@ -37,6 +37,7 @@ type ActionRun struct { TriggerUser *user_model.User `xorm:"-"` ScheduleID int64 Ref string `xorm:"index"` // the commit/tag/… that caused the run + IsRefDeleted bool `xorm:"-"` CommitSHA string IsForkPullRequest bool // If this is triggered by a PR from a forked repository or an untrusted user, we need to check if it is approved and limit permissions when running the workflow. NeedApproval bool // may need approval if it's a fork pull request diff --git a/models/git/branch.go b/models/git/branch.go index 74923f2b9b..702d767c75 100644 --- a/models/git/branch.go +++ b/models/git/branch.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" @@ -162,9 +163,22 @@ func GetBranch(ctx context.Context, repoID int64, branchName string) (*Branch, e return &branch, nil } -func GetBranches(ctx context.Context, repoID int64, branchNames []string) ([]*Branch, error) { +func GetBranches(ctx context.Context, repoID int64, branchNames []string, includeDeleted bool) ([]*Branch, error) { branches := make([]*Branch, 0, len(branchNames)) - return branches, db.GetEngine(ctx).Where("repo_id=?", repoID).In("name", branchNames).Find(&branches) + + sess := db.GetEngine(ctx).Where("repo_id=?", repoID).In("name", branchNames) + if !includeDeleted { + sess.And("is_deleted=?", false) + } + return branches, sess.Find(&branches) +} + +func BranchesToNamesSet(branches []*Branch) container.Set[string] { + names := make(container.Set[string], len(branches)) + for _, branch := range branches { + names.Add(branch.Name) + } + return names } func AddBranches(ctx context.Context, branches []*Branch) error { diff --git a/routers/web/repo/actions/actions.go b/routers/web/repo/actions/actions.go index ff3b16159b..283d476df1 100644 --- a/routers/web/repo/actions/actions.go +++ b/routers/web/repo/actions/actions.go @@ -12,11 +12,13 @@ import ( actions_model "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/actions" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -222,6 +224,10 @@ func List(ctx *context.Context) { return } + if err := loadIsRefDeleted(ctx, runs); err != nil { + log.Error("LoadIsRefDeleted", err) + } + ctx.Data["Runs"] = runs ctx.Data["Repo"] = ctx.Repo @@ -245,3 +251,31 @@ func List(ctx *context.Context) { ctx.HTML(http.StatusOK, tplListActions) } + +// loadIsRefDeleted loads the IsRefDeleted field for each run in the list. +// TODO: move this function to models/actions/run_list.go but now it will result in a circular import. +func loadIsRefDeleted(ctx *context.Context, runs actions_model.RunList) error { + branches := make(container.Set[string], len(runs)) + for _, run := range runs { + refName := git.RefName(run.Ref) + if refName.IsBranch() { + branches.Add(refName.ShortName()) + } + } + if len(branches) == 0 { + return nil + } + + branchInfos, err := git_model.GetBranches(ctx, ctx.Repo.Repository.ID, branches.Values(), false) + if err != nil { + return err + } + branchSet := git_model.BranchesToNamesSet(branchInfos) + for _, run := range runs { + refName := git.RefName(run.Ref) + if refName.IsBranch() && !branchSet.Contains(run.Ref) { + run.IsRefDeleted = true + } + } + return nil +} diff --git a/routers/web/repo/actions/view.go b/routers/web/repo/actions/view.go index cab37acdd1..dea31bb1c4 100644 --- a/routers/web/repo/actions/view.go +++ b/routers/web/repo/actions/view.go @@ -20,10 +20,13 @@ import ( actions_model "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/actions" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/templates" @@ -157,8 +160,9 @@ type ViewUser struct { } type ViewBranch struct { - Name string `json:"name"` - Link string `json:"link"` + Name string `json:"name"` + Link string `json:"link"` + IsDeleted bool `json:"isDeleted"` } type ViewJobStep struct { @@ -227,6 +231,16 @@ func ViewPost(ctx *context_module.Context) { Name: run.PrettyRef(), Link: run.RefLink(), } + refName := git.RefName(run.Ref) + if refName.IsBranch() { + b, err := git_model.GetBranch(ctx, ctx.Repo.Repository.ID, refName.ShortName()) + if err != nil && !git_model.IsErrBranchNotExist(err) { + log.Error("GetBranch: %v", err) + } else if git_model.IsErrBranchNotExist(err) || (b != nil && b.IsDeleted) { + branch.IsDeleted = true + } + } + resp.State.Run.Commit = ViewCommit{ LocaleCommit: ctx.Locale.TrString("actions.runs.commit"), LocalePushedBy: ctx.Locale.TrString("actions.runs.pushed_by"), diff --git a/services/repository/branch.go b/services/repository/branch.go index 87fe03c12d..8e1a6cd27f 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -254,7 +254,7 @@ func SyncBranchesToDB(ctx context.Context, repoID, pusherID int64, branchNames, } return db.WithTx(ctx, func(ctx context.Context) error { - branches, err := git_model.GetBranches(ctx, repoID, branchNames) + branches, err := git_model.GetBranches(ctx, repoID, branchNames, true) if err != nil { return fmt.Errorf("git_model.GetBranches: %v", err) } diff --git a/templates/repo/actions/runs_list.tmpl b/templates/repo/actions/runs_list.tmpl index ef764fa357..060fc1b66a 100644 --- a/templates/repo/actions/runs_list.tmpl +++ b/templates/repo/actions/runs_list.tmpl @@ -27,10 +27,10 @@
- {{if .RefLink}} - {{.PrettyRef}} + {{if .IsRefDeleted}} + {{.PrettyRef}} {{else}} - {{.PrettyRef}} + {{.PrettyRef}} {{end}}
{{svg "octicon-calendar" 16}}{{DateUtils.TimeSince .Updated}}
diff --git a/web_src/js/components/RepoActionView.vue b/web_src/js/components/RepoActionView.vue index 941440c6f3..13ac9427f3 100644 --- a/web_src/js/components/RepoActionView.vue +++ b/web_src/js/components/RepoActionView.vue @@ -444,7 +444,8 @@ export function initRepositoryActionView() { {{ run.commit.localePushedBy }} {{ run.commit.pusher.displayName }} - {{ run.commit.branch.name }} + {{ run.commit.branch.name }} + {{ run.commit.branch.name }}
From 7ceb30c000c4cbcb8b73292827d4ebfcb1bc92bd Mon Sep 17 00:00:00 2001 From: Rowan Bohde Date: Thu, 12 Dec 2024 15:10:47 -0600 Subject: [PATCH 05/11] Avoid MacOS keychain dialog in integration tests (#32813) Mac's git installation ships with a system wide config that configures the credential helper `osxkeychain`, which will prompt the user with a dialog. ``` $ git config list --system credential.helper=osxkeychain ``` By setting the environment variable [`GIT_CONFIG_NOSYSTEM=true`](https://git-scm.com/docs/git-config#ENVIRONMENT), Git will not load the system wide config, preventing the dialog from populating. Closes #26717 (cherry picked from commit a03fdd9566d62abd208af9ae30e58802a658e358) --- tests/integration/integration_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 606df2ed1c..65b715a688 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -166,6 +166,11 @@ func TestMain(m *testing.M) { os.Unsetenv("GIT_COMMITTER_EMAIL") os.Unsetenv("GIT_COMMITTER_DATE") + // Avoid loading the default system config. On MacOS, this config + // sets the osxkeychain credential helper, which will cause tests + // to freeze with a dialog. + os.Setenv("GIT_CONFIG_NOSYSTEM", "true") + err := unittest.InitFixtures( unittest.FixturesOptions{ Dir: filepath.Join(filepath.Dir(setting.AppPath), "models/fixtures/"), From 4be37a986e28a96c932ea7bf0b43139cb54750d0 Mon Sep 17 00:00:00 2001 From: hiifong Date: Fri, 13 Dec 2024 05:36:39 +0800 Subject: [PATCH 06/11] Fix lfs migration (#32812) Fix: #32803 (cherry picked from commit 0b8a8941a01ed4bf914843c88740ad6203550b85) --- modules/lfs/http_client.go | 1 + modules/lfs/shared.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 3060e25754..50f0e7a8d8 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -236,6 +236,7 @@ func createRequest(ctx context.Context, method, url string, headers map[string]s req.Header.Set(key, value) } req.Header.Set("Accept", AcceptHeader) + req.Header.Set("User-Agent", UserAgentHeader) return req, nil } diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index a4326b57b2..40ad789c1d 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -15,7 +15,8 @@ const ( // MediaType contains the media type for LFS server requests MediaType = "application/vnd.git-lfs+json" // Some LFS servers offer content with other types, so fallback to '*/*' if application/vnd.git-lfs+json cannot be served - AcceptHeader = "application/vnd.git-lfs+json;q=0.9, */*;q=0.8" + AcceptHeader = "application/vnd.git-lfs+json;q=0.9, */*;q=0.8" + UserAgentHeader = "git-lfs" ) // BatchRequest contains multiple requests processed in one batch operation. From 96a7f0a3f065c5db8fdf352c93c8367e24d259de Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Sat, 14 Dec 2024 10:22:30 +0800 Subject: [PATCH 07/11] Fix missing outputs for jobs with matrix (#32823) Fix #32795 If a job uses a matrix, multiple `ActionRunJobs` may have the same `JobID`. We need to merge the outputs of these jobs to make them available to the jobs that need them. (cherry picked from commit 7269130d2878d51dcdf11f7081a591f85bd493e8) Conflicts: models/fixtures/action_run.yml models/fixtures/action_run_job.yml trivial context conflicts --- models/actions/run_job.go | 4 +- models/fixtures/action_run.yml | 19 ++++++++ models/fixtures/action_run_job.yml | 43 +++++++++++++++++ models/fixtures/action_task.yml | 60 ++++++++++++++++++++++++ models/fixtures/action_task_output.yml | 20 ++++++++ routers/api/actions/runner/main_test.go | 16 +++++++ routers/api/actions/runner/utils.go | 60 +++++++++++++++++------- routers/api/actions/runner/utils_test.go | 29 ++++++++++++ 8 files changed, 233 insertions(+), 18 deletions(-) create mode 100644 models/fixtures/action_task_output.yml create mode 100644 routers/api/actions/runner/main_test.go create mode 100644 routers/api/actions/runner/utils_test.go diff --git a/models/actions/run_job.go b/models/actions/run_job.go index 4b8664077d..2319af8e08 100644 --- a/models/actions/run_job.go +++ b/models/actions/run_job.go @@ -137,7 +137,7 @@ func UpdateRunJob(ctx context.Context, job *ActionRunJob, cond builder.Cond, col if err != nil { return 0, err } - run.Status = aggregateJobStatus(jobs) + run.Status = AggregateJobStatus(jobs) if run.Started.IsZero() && run.Status.IsRunning() { run.Started = timeutil.TimeStampNow() } @@ -152,7 +152,7 @@ func UpdateRunJob(ctx context.Context, job *ActionRunJob, cond builder.Cond, col return affected, nil } -func aggregateJobStatus(jobs []*ActionRunJob) Status { +func AggregateJobStatus(jobs []*ActionRunJob) Status { allDone := true allWaiting := true hasFailure := false diff --git a/models/fixtures/action_run.yml b/models/fixtures/action_run.yml index 9c60b352f9..2fe9094d13 100644 --- a/models/fixtures/action_run.yml +++ b/models/fixtures/action_run.yml @@ -413,6 +413,25 @@ }, "total_commits": 0 } +- + id: 793 + title: "job output" + repo_id: 4 + owner_id: 1 + workflow_id: "test.yaml" + index: 189 + trigger_user_id: 1 + ref: "refs/heads/master" + commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0" + event: "push" + is_fork_pull_request: 0 + status: 1 + started: 1683636528 + stopped: 1683636626 + created: 1683636108 + updated: 1683636626 + need_approval: 0 + approved_by: 0 - id: 891 title: "update actions" diff --git a/models/fixtures/action_run_job.yml b/models/fixtures/action_run_job.yml index 0b02d0e17e..117bb5ea05 100644 --- a/models/fixtures/action_run_job.yml +++ b/models/fixtures/action_run_job.yml @@ -26,6 +26,49 @@ status: 1 started: 1683636528 stopped: 1683636626 +- + id: 194 + run_id: 793 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + is_fork_pull_request: 0 + name: job1 (1) + attempt: 1 + job_id: job1 + task_id: 49 + status: 1 + started: 1683636528 + stopped: 1683636626 +- + id: 195 + run_id: 793 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + is_fork_pull_request: 0 + name: job1 (2) + attempt: 1 + job_id: job1 + task_id: 50 + status: 1 + started: 1683636528 + stopped: 1683636626 +- + id: 196 + run_id: 793 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + is_fork_pull_request: 0 + name: job2 + attempt: 1 + job_id: job2 + needs: [job1] + task_id: 51 + status: 5 + started: 1683636528 + stopped: 1683636626 - id: 292 run_id: 891 diff --git a/models/fixtures/action_task.yml b/models/fixtures/action_task.yml index d88a8ed8a9..506a47d8a0 100644 --- a/models/fixtures/action_task.yml +++ b/models/fixtures/action_task.yml @@ -57,3 +57,63 @@ log_length: 707 log_size: 90179 log_expired: 0 +- + id: 49 + job_id: 194 + attempt: 1 + runner_id: 1 + status: 1 # success + started: 1683636528 + stopped: 1683636626 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + is_fork_pull_request: 0 + token_hash: b8d3962425466b6709b9ac51446f93260c54afe8e7b6d3686e34f991fb8a8953822b0deed86fe41a103f34bc48dbc4784220 + token_salt: ffffffffff + token_last_eight: ffffffff + log_filename: artifact-test2/2f/47.log + log_in_storage: 1 + log_length: 707 + log_size: 90179 + log_expired: 0 +- + id: 50 + job_id: 195 + attempt: 1 + runner_id: 1 + status: 1 # success + started: 1683636528 + stopped: 1683636626 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + is_fork_pull_request: 0 + token_hash: b8d3962425466b6709b9ac51446f93260c54afe8e7b6d3686e34f991fb8a8953822b0deed86fe41a103f34bc48dbc4784221 + token_salt: ffffffffff + token_last_eight: ffffffff + log_filename: artifact-test2/2f/47.log + log_in_storage: 1 + log_length: 707 + log_size: 90179 + log_expired: 0 +- + id: 51 + job_id: 196 + attempt: 1 + runner_id: 1 + status: 6 # running + started: 1683636528 + stopped: 1683636626 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + is_fork_pull_request: 0 + token_hash: b8d3962425466b6709b9ac51446f93260c54afe8e7b6d3686e34f991fb8a8953822b0deed86fe41a103f34bc48dbc4784222 + token_salt: ffffffffff + token_last_eight: ffffffff + log_filename: artifact-test2/2f/47.log + log_in_storage: 1 + log_length: 707 + log_size: 90179 + log_expired: 0 diff --git a/models/fixtures/action_task_output.yml b/models/fixtures/action_task_output.yml new file mode 100644 index 0000000000..314e9f7115 --- /dev/null +++ b/models/fixtures/action_task_output.yml @@ -0,0 +1,20 @@ +- + id: 1 + task_id: 49 + output_key: output_a + output_value: abc +- + id: 2 + task_id: 49 + output_key: output_b + output_value: '' +- + id: 3 + task_id: 50 + output_key: output_a + output_value: '' +- + id: 4 + task_id: 50 + output_key: output_b + output_value: bbb diff --git a/routers/api/actions/runner/main_test.go b/routers/api/actions/runner/main_test.go new file mode 100644 index 0000000000..bed63c166e --- /dev/null +++ b/routers/api/actions/runner/main_test.go @@ -0,0 +1,16 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package runner + +import ( + "testing" + + "code.gitea.io/gitea/models/unittest" + + _ "code.gitea.io/gitea/models/forgefed" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m) +} diff --git a/routers/api/actions/runner/utils.go b/routers/api/actions/runner/utils.go index ff6ec5bd54..539be8d889 100644 --- a/routers/api/actions/runner/utils.go +++ b/routers/api/actions/runner/utils.go @@ -162,28 +162,56 @@ func findTaskNeeds(ctx context.Context, task *actions_model.ActionTask) (map[str return nil, fmt.Errorf("FindRunJobs: %w", err) } - ret := make(map[string]*runnerv1.TaskNeed, len(needs)) + jobIDJobs := make(map[string][]*actions_model.ActionRunJob) for _, job := range jobs { - if !needs.Contains(job.JobID) { + jobIDJobs[job.JobID] = append(jobIDJobs[job.JobID], job) + } + + ret := make(map[string]*runnerv1.TaskNeed, len(needs)) + for jobID, jobsWithSameID := range jobIDJobs { + if !needs.Contains(jobID) { continue } - if job.TaskID == 0 || !job.Status.IsDone() { - // it shouldn't happen, or the job has been rerun - continue + var jobOutputs map[string]string + for _, job := range jobsWithSameID { + if job.TaskID == 0 || !job.Status.IsDone() { + // it shouldn't happen, or the job has been rerun + continue + } + got, err := actions_model.FindTaskOutputByTaskID(ctx, job.TaskID) + if err != nil { + return nil, fmt.Errorf("FindTaskOutputByTaskID: %w", err) + } + outputs := make(map[string]string, len(got)) + for _, v := range got { + outputs[v.OutputKey] = v.OutputValue + } + if len(jobOutputs) == 0 { + jobOutputs = outputs + } else { + jobOutputs = mergeTwoOutputs(outputs, jobOutputs) + } } - outputs := make(map[string]string) - got, err := actions_model.FindTaskOutputByTaskID(ctx, job.TaskID) - if err != nil { - return nil, fmt.Errorf("FindTaskOutputByTaskID: %w", err) - } - for _, v := range got { - outputs[v.OutputKey] = v.OutputValue - } - ret[job.JobID] = &runnerv1.TaskNeed{ - Outputs: outputs, - Result: runnerv1.Result(job.Status), + ret[jobID] = &runnerv1.TaskNeed{ + Outputs: jobOutputs, + Result: runnerv1.Result(actions_model.AggregateJobStatus(jobsWithSameID)), } } return ret, nil } + +// mergeTwoOutputs merges two outputs from two different ActionRunJobs +// Values with the same output name may be overridden. The user should ensure the output names are unique. +// See https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#using-job-outputs-in-a-matrix-job +func mergeTwoOutputs(o1, o2 map[string]string) map[string]string { + ret := make(map[string]string, len(o1)) + for k1, v1 := range o1 { + if len(v1) > 0 { + ret[k1] = v1 + } else { + ret[k1] = o2[k1] + } + } + return ret +} diff --git a/routers/api/actions/runner/utils_test.go b/routers/api/actions/runner/utils_test.go new file mode 100644 index 0000000000..c8a0a28d65 --- /dev/null +++ b/routers/api/actions/runner/utils_test.go @@ -0,0 +1,29 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package runner + +import ( + "context" + "testing" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/unittest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_findTaskNeeds(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + + task := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionTask{ID: 51}) + + ret, err := findTaskNeeds(context.Background(), task) + require.NoError(t, err) + assert.Len(t, ret, 1) + assert.Contains(t, ret, "job1") + assert.Len(t, ret["job1"].Outputs, 2) + assert.Equal(t, "abc", ret["job1"].Outputs["output_a"]) + assert.Equal(t, "bbb", ret["job1"].Outputs["output_b"]) +} From 14a5d177e1e16225c79f63aff795dfcd5ebd2007 Mon Sep 17 00:00:00 2001 From: hiifong Date: Sat, 14 Dec 2024 11:35:19 +0800 Subject: [PATCH 08/11] Add User-Agent for gitea's self-implemented lfs client. (#32832) (cherry picked from commit 82c59d52ea650ce42bbca2c6740d9449d06e77be) --- modules/lfs/shared.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index 40ad789c1d..cd9488e3db 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -14,9 +14,12 @@ import ( const ( // MediaType contains the media type for LFS server requests MediaType = "application/vnd.git-lfs+json" - // Some LFS servers offer content with other types, so fallback to '*/*' if application/vnd.git-lfs+json cannot be served - AcceptHeader = "application/vnd.git-lfs+json;q=0.9, */*;q=0.8" - UserAgentHeader = "git-lfs" + // AcceptHeader Some LFS servers offer content with other types, so fallback to '*/*' if application/vnd.git-lfs+json cannot be served + AcceptHeader = "application/vnd.git-lfs+json;q=0.9, */*;q=0.8" + // UserAgentHeader Add User-Agent for gitea's self-implemented lfs client, + // and the version is consistent with the latest version of git lfs can be avoided incompatibilities. + // Some lfs servers will check this + UserAgentHeader = "git-lfs/3.6.0 (Gitea)" ) // BatchRequest contains multiple requests processed in one batch operation. From 68278c796de06f503bd2ecdacda3bec77f2c0d04 Mon Sep 17 00:00:00 2001 From: hiifong Date: Sun, 15 Dec 2024 10:06:21 +0800 Subject: [PATCH 09/11] In some lfs server implementations, they require the ref attribute. (#32838) Fix: #32611 In some lfs server implementations, they require the ref attribute. --------- Co-authored-by: wxiaoguang (cherry picked from commit 7616aeb2ea2a02c15480dcd4a232e98081569690) --- modules/lfs/http_client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 50f0e7a8d8..3acd23b8f7 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -72,7 +72,10 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin url := fmt.Sprintf("%s/objects/batch", c.endpoint) - request := &BatchRequest{operation, c.transferNames(), nil, objects} + // `ref` is an "optional object describing the server ref that the objects belong to" + // but some (incorrect) lfs servers require it, so maybe adding an empty ref here doesn't break the correct ones. + // https://github.com/git-lfs/git-lfs/blob/a32a02b44bf8a511aa14f047627c49e1a7fd5021/docs/api/batch.md?plain=1#L37 + request := &BatchRequest{operation, c.transferNames(), &Reference{}, objects} payload := new(bytes.Buffer) err := json.NewEncoder(payload).Encode(request) if err != nil { From 7ecbd4303e7fbb3e5059c1614965c4c27eeecbd5 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Sun, 15 Dec 2024 10:16:04 +0100 Subject: [PATCH 10/11] chore(release-notes): notes for the week 2024-51 weekly cherry pick --- release-notes/6271.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 release-notes/6271.md diff --git a/release-notes/6271.md b/release-notes/6271.md new file mode 100644 index 0000000000..4a4821682f --- /dev/null +++ b/release-notes/6271.md @@ -0,0 +1,4 @@ +fix: [commit](https://codeberg.org/forgejo/forgejo/commit/96a7f0a3f065c5db8fdf352c93c8367e24d259de) Fix missing outputs for jobs with matrix +fix: [commit](https://codeberg.org/forgejo/forgejo/commit/2b5c69c451a684b20119e2521dc23734c7869241) Detect whether action view branch was deleted +feat: [commit](https://codeberg.org/forgejo/forgejo/commit/b0d6a7f07bff836190a8e87fe5645d5557893e32) Implement update branch API +fix: [commit](https://codeberg.org/forgejo/forgejo/commit/bf934c96c92d643678ac7a18697b6563bc9d20a5) Add standard-compliant route to serve outdated R packages From 9283b03f3ead28526cd473299048f881164a965e Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 17 Dec 2024 06:18:24 +0100 Subject: [PATCH 11/11] Add User-Agent for gitea's self-implemented lfs client. (branding) --- modules/lfs/shared.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index cd9488e3db..ae4bb1f86b 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -19,7 +19,7 @@ const ( // UserAgentHeader Add User-Agent for gitea's self-implemented lfs client, // and the version is consistent with the latest version of git lfs can be avoided incompatibilities. // Some lfs servers will check this - UserAgentHeader = "git-lfs/3.6.0 (Gitea)" + UserAgentHeader = "git-lfs/3.6.0 (Forgejo)" ) // BatchRequest contains multiple requests processed in one batch operation.