2021-06-14 12:20:43 -05:00
|
|
|
// Copyright 2021 The Gitea Authors. All rights reserved.
|
2022-11-27 13:20:29 -05:00
|
|
|
// SPDX-License-Identifier: MIT
|
2021-06-14 12:20:43 -05:00
|
|
|
|
|
|
|
package mirror
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"errors"
|
2021-11-30 15:06:32 -05:00
|
|
|
"fmt"
|
2021-06-14 12:20:43 -05:00
|
|
|
"io"
|
|
|
|
"regexp"
|
2022-03-27 06:54:09 -05:00
|
|
|
"strings"
|
2021-06-14 12:20:43 -05:00
|
|
|
"time"
|
|
|
|
|
2023-12-25 15:25:29 -05:00
|
|
|
"code.gitea.io/gitea/models/db"
|
2021-12-09 20:27:50 -05:00
|
|
|
repo_model "code.gitea.io/gitea/models/repo"
|
2021-06-14 12:20:43 -05:00
|
|
|
"code.gitea.io/gitea/modules/git"
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-27 15:09:51 -05:00
|
|
|
"code.gitea.io/gitea/modules/gitrepo"
|
2021-06-14 12:20:43 -05:00
|
|
|
"code.gitea.io/gitea/modules/lfs"
|
|
|
|
"code.gitea.io/gitea/modules/log"
|
2021-11-30 15:06:32 -05:00
|
|
|
"code.gitea.io/gitea/modules/process"
|
2021-06-14 12:20:43 -05:00
|
|
|
"code.gitea.io/gitea/modules/repository"
|
|
|
|
"code.gitea.io/gitea/modules/setting"
|
|
|
|
"code.gitea.io/gitea/modules/timeutil"
|
|
|
|
"code.gitea.io/gitea/modules/util"
|
|
|
|
)
|
|
|
|
|
|
|
|
var stripExitStatus = regexp.MustCompile(`exit status \d+ - `)
|
|
|
|
|
|
|
|
// AddPushMirrorRemote registers the push mirror remote.
|
2024-05-29 12:52:26 -05:00
|
|
|
var AddPushMirrorRemote = addPushMirrorRemote
|
|
|
|
|
|
|
|
func addPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr string) error {
|
2021-06-14 12:20:43 -05:00
|
|
|
addRemoteAndConfig := func(addr, path string) error {
|
2022-10-23 09:44:45 -05:00
|
|
|
cmd := git.NewCommand(ctx, "remote", "add", "--mirror=push").AddDynamicArguments(m.RemoteName, addr)
|
2022-03-27 06:54:09 -05:00
|
|
|
if strings.Contains(addr, "://") && strings.Contains(addr, "@") {
|
2022-03-30 21:25:40 -05:00
|
|
|
cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, util.SanitizeCredentialURLs(addr), path))
|
2022-03-27 06:54:09 -05:00
|
|
|
} else {
|
|
|
|
cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, addr, path))
|
|
|
|
}
|
2022-03-31 21:55:30 -05:00
|
|
|
if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: path}); err != nil {
|
2021-06-14 12:20:43 -05:00
|
|
|
return err
|
|
|
|
}
|
Refactor git command package to improve security and maintainability (#22678)
This PR follows #21535 (and replace #22592)
## Review without space diff
https://github.com/go-gitea/gitea/pull/22678/files?diff=split&w=1
## Purpose of this PR
1. Make git module command completely safe (risky user inputs won't be
passed as argument option anymore)
2. Avoid low-level mistakes like
https://github.com/go-gitea/gitea/pull/22098#discussion_r1045234918
3. Remove deprecated and dirty `CmdArgCheck` function, hide the `CmdArg`
type
4. Simplify code when using git command
## The main idea of this PR
* Move the `git.CmdArg` to the `internal` package, then no other package
except `git` could use it. Then developers could never do
`AddArguments(git.CmdArg(userInput))` any more.
* Introduce `git.ToTrustedCmdArgs`, it's for user-provided and already
trusted arguments. It's only used in a few cases, for example: use git
arguments from config file, help unit test with some arguments.
* Introduce `AddOptionValues` and `AddOptionFormat`, they make code more
clear and simple:
* Before: `AddArguments("-m").AddDynamicArguments(message)`
* After: `AddOptionValues("-m", message)`
* -
* Before: `AddArguments(git.CmdArg(fmt.Sprintf("--author='%s <%s>'",
sig.Name, sig.Email)))`
* After: `AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email)`
## FAQ
### Why these changes were not done in #21535 ?
#21535 is mainly a search&replace, it did its best to not change too
much logic.
Making the framework better needs a lot of changes, so this separate PR
is needed as the second step.
### The naming of `AddOptionXxx`
According to git's manual, the `--xxx` part is called `option`.
### How can it guarantee that `internal.CmdArg` won't be not misused?
Go's specification guarantees that. Trying to access other package's
internal package causes compilation error.
And, `golangci-lint` also denies the git/internal package. Only the
`git/command.go` can use it carefully.
### There is still a `ToTrustedCmdArgs`, will it still allow developers
to make mistakes and pass untrusted arguments?
Generally speaking, no. Because when using `ToTrustedCmdArgs`, the code
will be very complex (see the changes for examples). Then developers and
reviewers can know that something might be unreasonable.
### Why there was a `CmdArgCheck` and why it's removed?
At the moment of #21535, to reduce unnecessary changes, `CmdArgCheck`
was introduced as a hacky patch. Now, almost all code could be written
as `cmd := NewCommand(); cmd.AddXxx(...)`, then there is no need for
`CmdArgCheck` anymore.
### Why many codes for `signArg == ""` is deleted?
Because in the old code, `signArg` could never be empty string, it's
either `-S[key-id]` or `--no-gpg-sign`. So the `signArg == ""` is just
dead code.
---------
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2023-02-03 21:30:43 -05:00
|
|
|
if _, _, err := git.NewCommand(ctx, "config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunStdString(&git.RunOpts{Dir: path}); err != nil {
|
2021-06-14 12:20:43 -05:00
|
|
|
return err
|
|
|
|
}
|
Refactor git command package to improve security and maintainability (#22678)
This PR follows #21535 (and replace #22592)
## Review without space diff
https://github.com/go-gitea/gitea/pull/22678/files?diff=split&w=1
## Purpose of this PR
1. Make git module command completely safe (risky user inputs won't be
passed as argument option anymore)
2. Avoid low-level mistakes like
https://github.com/go-gitea/gitea/pull/22098#discussion_r1045234918
3. Remove deprecated and dirty `CmdArgCheck` function, hide the `CmdArg`
type
4. Simplify code when using git command
## The main idea of this PR
* Move the `git.CmdArg` to the `internal` package, then no other package
except `git` could use it. Then developers could never do
`AddArguments(git.CmdArg(userInput))` any more.
* Introduce `git.ToTrustedCmdArgs`, it's for user-provided and already
trusted arguments. It's only used in a few cases, for example: use git
arguments from config file, help unit test with some arguments.
* Introduce `AddOptionValues` and `AddOptionFormat`, they make code more
clear and simple:
* Before: `AddArguments("-m").AddDynamicArguments(message)`
* After: `AddOptionValues("-m", message)`
* -
* Before: `AddArguments(git.CmdArg(fmt.Sprintf("--author='%s <%s>'",
sig.Name, sig.Email)))`
* After: `AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email)`
## FAQ
### Why these changes were not done in #21535 ?
#21535 is mainly a search&replace, it did its best to not change too
much logic.
Making the framework better needs a lot of changes, so this separate PR
is needed as the second step.
### The naming of `AddOptionXxx`
According to git's manual, the `--xxx` part is called `option`.
### How can it guarantee that `internal.CmdArg` won't be not misused?
Go's specification guarantees that. Trying to access other package's
internal package causes compilation error.
And, `golangci-lint` also denies the git/internal package. Only the
`git/command.go` can use it carefully.
### There is still a `ToTrustedCmdArgs`, will it still allow developers
to make mistakes and pass untrusted arguments?
Generally speaking, no. Because when using `ToTrustedCmdArgs`, the code
will be very complex (see the changes for examples). Then developers and
reviewers can know that something might be unreasonable.
### Why there was a `CmdArgCheck` and why it's removed?
At the moment of #21535, to reduce unnecessary changes, `CmdArgCheck`
was introduced as a hacky patch. Now, almost all code could be written
as `cmd := NewCommand(); cmd.AddXxx(...)`, then there is no need for
`CmdArgCheck` anymore.
### Why many codes for `signArg == ""` is deleted?
Because in the old code, `signArg` could never be empty string, it's
either `-S[key-id]` or `--no-gpg-sign`. So the `signArg == ""` is just
dead code.
---------
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2023-02-03 21:30:43 -05:00
|
|
|
if _, _, err := git.NewCommand(ctx, "config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunStdString(&git.RunOpts{Dir: path}); err != nil {
|
2021-06-14 12:20:43 -05:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := addRemoteAndConfig(addr, m.Repo.RepoPath()); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if m.Repo.HasWiki() {
|
2022-01-19 18:26:57 -05:00
|
|
|
wikiRemoteURL := repository.WikiRemoteURL(ctx, addr)
|
2021-06-14 12:20:43 -05:00
|
|
|
if len(wikiRemoteURL) > 0 {
|
|
|
|
if err := addRemoteAndConfig(wikiRemoteURL, m.Repo.WikiPath()); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// RemovePushMirrorRemote removes the push mirror remote.
|
2022-01-19 18:26:57 -05:00
|
|
|
func RemovePushMirrorRemote(ctx context.Context, m *repo_model.PushMirror) error {
|
2022-10-23 09:44:45 -05:00
|
|
|
cmd := git.NewCommand(ctx, "remote", "rm").AddDynamicArguments(m.RemoteName)
|
2023-10-03 05:30:41 -05:00
|
|
|
_ = m.GetRepository(ctx)
|
2021-06-14 12:20:43 -05:00
|
|
|
|
2022-03-31 21:55:30 -05:00
|
|
|
if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.RepoPath()}); err != nil {
|
2021-06-14 12:20:43 -05:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if m.Repo.HasWiki() {
|
2022-03-31 21:55:30 -05:00
|
|
|
if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.WikiPath()}); err != nil {
|
2021-06-14 12:20:43 -05:00
|
|
|
// The wiki remote may not exist
|
|
|
|
log.Warn("Wiki Remote[%d] could not be removed: %v", m.ID, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// SyncPushMirror starts the sync of the push mirror and schedules the next run.
|
|
|
|
func SyncPushMirror(ctx context.Context, mirrorID int64) bool {
|
|
|
|
log.Trace("SyncPushMirror [mirror: %d]", mirrorID)
|
|
|
|
defer func() {
|
|
|
|
err := recover()
|
|
|
|
if err == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// There was a panic whilst syncPushMirror...
|
|
|
|
log.Error("PANIC whilst syncPushMirror[%d] Panic: %v\nStacktrace: %s", mirrorID, err, log.Stack(2))
|
|
|
|
}()
|
|
|
|
|
2023-12-25 15:25:29 -05:00
|
|
|
// TODO: Handle "!exist" better
|
|
|
|
m, exist, err := db.GetByID[repo_model.PushMirror](ctx, mirrorID)
|
|
|
|
if err != nil || !exist {
|
2021-06-14 12:20:43 -05:00
|
|
|
log.Error("GetPushMirrorByID [%d]: %v", mirrorID, err)
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2023-10-03 05:30:41 -05:00
|
|
|
_ = m.GetRepository(ctx)
|
2022-05-20 09:08:52 -05:00
|
|
|
|
2021-06-14 12:20:43 -05:00
|
|
|
m.LastError = ""
|
|
|
|
|
2021-11-30 15:06:32 -05:00
|
|
|
ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Syncing PushMirror %s/%s to %s", m.Repo.OwnerName, m.Repo.Name, m.RemoteName))
|
|
|
|
defer finished()
|
|
|
|
|
2021-06-14 12:20:43 -05:00
|
|
|
log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Running Sync", m.ID, m.Repo)
|
|
|
|
err = runPushSync(ctx, m)
|
|
|
|
if err != nil {
|
|
|
|
log.Error("SyncPushMirror [mirror: %d][repo: %-v]: %v", m.ID, m.Repo, err)
|
|
|
|
m.LastError = stripExitStatus.ReplaceAllLiteralString(err.Error(), "")
|
|
|
|
}
|
|
|
|
|
|
|
|
m.LastUpdateUnix = timeutil.TimeStampNow()
|
|
|
|
|
2022-07-30 11:45:59 -05:00
|
|
|
if err := repo_model.UpdatePushMirror(ctx, m); err != nil {
|
2021-06-14 12:20:43 -05:00
|
|
|
log.Error("UpdatePushMirror [%d]: %v", m.ID, err)
|
|
|
|
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Finished", m.ID, m.Repo)
|
|
|
|
|
|
|
|
return err == nil
|
|
|
|
}
|
|
|
|
|
2021-12-09 20:27:50 -05:00
|
|
|
func runPushSync(ctx context.Context, m *repo_model.PushMirror) error {
|
2021-06-14 12:20:43 -05:00
|
|
|
timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
|
|
|
|
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-27 15:09:51 -05:00
|
|
|
performPush := func(repo *repo_model.Repository, isWiki bool) error {
|
|
|
|
path := repo.RepoPath()
|
|
|
|
if isWiki {
|
|
|
|
path = repo.WikiPath()
|
|
|
|
}
|
2022-06-11 08:50:14 -05:00
|
|
|
remoteURL, err := git.GetRemoteURL(ctx, path, m.RemoteName)
|
2021-06-14 12:20:43 -05:00
|
|
|
if err != nil {
|
|
|
|
log.Error("GetRemoteAddress(%s) Error %v", path, err)
|
|
|
|
return errors.New("Unexpected error")
|
|
|
|
}
|
|
|
|
|
|
|
|
if setting.LFS.StartServer {
|
|
|
|
log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
|
|
|
|
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-27 15:09:51 -05:00
|
|
|
var gitRepo *git.Repository
|
|
|
|
if isWiki {
|
|
|
|
gitRepo, err = gitrepo.OpenWikiRepository(ctx, repo)
|
|
|
|
} else {
|
|
|
|
gitRepo, err = gitrepo.OpenRepository(ctx, repo)
|
|
|
|
}
|
2021-06-14 12:20:43 -05:00
|
|
|
if err != nil {
|
|
|
|
log.Error("OpenRepository: %v", err)
|
|
|
|
return errors.New("Unexpected error")
|
|
|
|
}
|
|
|
|
defer gitRepo.Close()
|
|
|
|
|
2022-06-11 08:50:14 -05:00
|
|
|
endpoint := lfs.DetermineEndpoint(remoteURL.String(), "")
|
2021-11-20 04:34:05 -05:00
|
|
|
lfsClient := lfs.NewClient(endpoint, nil)
|
|
|
|
if err := pushAllLFSObjects(ctx, gitRepo, lfsClient); err != nil {
|
2022-03-30 21:25:40 -05:00
|
|
|
return util.SanitizeErrorCredentialURLs(err)
|
2021-06-14 12:20:43 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName)
|
|
|
|
|
2021-11-30 15:06:32 -05:00
|
|
|
if err := git.Push(ctx, path, git.PushOptions{
|
2021-06-14 12:20:43 -05:00
|
|
|
Remote: m.RemoteName,
|
|
|
|
Force: true,
|
|
|
|
Mirror: true,
|
|
|
|
Timeout: timeout,
|
|
|
|
}); err != nil {
|
|
|
|
log.Error("Error pushing %s mirror[%d] remote %s: %v", path, m.ID, m.RemoteName, err)
|
|
|
|
|
2022-03-30 21:25:40 -05:00
|
|
|
return util.SanitizeErrorCredentialURLs(err)
|
2021-06-14 12:20:43 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-27 15:09:51 -05:00
|
|
|
err := performPush(m.Repo, false)
|
2021-06-14 12:20:43 -05:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if m.Repo.HasWiki() {
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-27 15:09:51 -05:00
|
|
|
_, err := git.GetRemoteAddress(ctx, m.Repo.WikiPath(), m.RemoteName)
|
2021-06-14 12:20:43 -05:00
|
|
|
if err == nil {
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-27 15:09:51 -05:00
|
|
|
err := performPush(m.Repo, true)
|
2021-06-14 12:20:43 -05:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
log.Trace("Skipping wiki: No remote configured")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-11-20 04:34:05 -05:00
|
|
|
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, lfsClient lfs.Client) error {
|
2021-06-14 12:20:43 -05:00
|
|
|
contentStore := lfs.NewContentStore()
|
|
|
|
|
|
|
|
pointerChan := make(chan lfs.PointerBlob)
|
|
|
|
errChan := make(chan error, 1)
|
|
|
|
go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan)
|
|
|
|
|
|
|
|
uploadObjects := func(pointers []lfs.Pointer) error {
|
2021-11-20 04:34:05 -05:00
|
|
|
err := lfsClient.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) {
|
2021-06-14 12:20:43 -05:00
|
|
|
if objectError != nil {
|
|
|
|
return nil, objectError
|
|
|
|
}
|
|
|
|
|
|
|
|
content, err := contentStore.Get(p)
|
|
|
|
if err != nil {
|
|
|
|
log.Error("Error reading LFS object %v: %v", p, err)
|
|
|
|
}
|
|
|
|
return content, err
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return nil
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
var batch []lfs.Pointer
|
|
|
|
for pointerBlob := range pointerChan {
|
|
|
|
exists, err := contentStore.Exists(pointerBlob.Pointer)
|
|
|
|
if err != nil {
|
|
|
|
log.Error("Error checking if LFS object %v exists: %v", pointerBlob.Pointer, err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if !exists {
|
|
|
|
log.Trace("Skipping missing LFS object %v", pointerBlob.Pointer)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
batch = append(batch, pointerBlob.Pointer)
|
2021-11-20 04:34:05 -05:00
|
|
|
if len(batch) >= lfsClient.BatchSize() {
|
2021-06-14 12:20:43 -05:00
|
|
|
if err := uploadObjects(batch); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
batch = nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if len(batch) > 0 {
|
|
|
|
if err := uploadObjects(batch); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
err, has := <-errChan
|
|
|
|
if has {
|
|
|
|
log.Error("Error enumerating LFS objects for repository: %v", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
2023-08-26 21:24:45 -05:00
|
|
|
|
|
|
|
func syncPushMirrorWithSyncOnCommit(ctx context.Context, repoID int64) {
|
|
|
|
pushMirrors, err := repo_model.GetPushMirrorsSyncedOnCommit(ctx, repoID)
|
|
|
|
if err != nil {
|
|
|
|
log.Error("repo_model.GetPushMirrorsSyncedOnCommit failed: %v", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, mirror := range pushMirrors {
|
|
|
|
AddPushMirrorToQueue(mirror.ID)
|
|
|
|
}
|
|
|
|
}
|