mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-12-25 00:44:15 -05:00
Asynchronously populate the repo indexer (#3366)
* Populate repo indexer in background * Check if no repos exist * race cond
This commit is contained in:
parent
a8325dd1cb
commit
ccdb94992d
1 changed files with 41 additions and 17 deletions
|
@ -70,34 +70,58 @@ func InitRepoIndexer() {
|
||||||
if !setting.Indexer.RepoIndexerEnabled {
|
if !setting.Indexer.RepoIndexerEnabled {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
indexer.InitRepoIndexer(populateRepoIndexer)
|
|
||||||
repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
|
repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
|
||||||
|
indexer.InitRepoIndexer(populateRepoIndexerAsynchronously)
|
||||||
go processRepoIndexerOperationQueue()
|
go processRepoIndexerOperationQueue()
|
||||||
}
|
}
|
||||||
|
|
||||||
// populateRepoIndexer populate the repo indexer with data
|
// populateRepoIndexerAsynchronously asynchronously populates the repo indexer
|
||||||
func populateRepoIndexer() error {
|
// with pre-existing data. This should only be run when the indexer is created
|
||||||
log.Info("Populating repository indexer (this may take a while)")
|
// for the first time.
|
||||||
for page := 1; ; page++ {
|
func populateRepoIndexerAsynchronously() error {
|
||||||
repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
|
exist, err := x.Table("repository").Exist()
|
||||||
Page: page,
|
|
||||||
PageSize: RepositoryListDefaultPageSize,
|
|
||||||
OrderBy: SearchOrderByID,
|
|
||||||
Private: true,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
} else if len(repos) == 0 {
|
} else if !exist {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var maxRepoID int64
|
||||||
|
if _, err = x.Select("MAX(id)").Table("repository").Get(&maxRepoID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
go populateRepoIndexer(maxRepoID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// populateRepoIndexer populate the repo indexer with pre-existing data. This
|
||||||
|
// should only be run when the indexer is created for the first time.
|
||||||
|
func populateRepoIndexer(maxRepoID int64) {
|
||||||
|
log.Info("Populating the repo indexer with existing repositories")
|
||||||
|
// start with the maximum existing repo ID and work backwards, so that we
|
||||||
|
// don't include repos that are created after gitea starts; such repos will
|
||||||
|
// already be added to the indexer, and we don't need to add them again.
|
||||||
|
for maxRepoID > 0 {
|
||||||
|
repos := make([]*Repository, 0, RepositoryListDefaultPageSize)
|
||||||
|
err := x.Where("id <= ?", maxRepoID).
|
||||||
|
OrderBy("id DESC").
|
||||||
|
Limit(RepositoryListDefaultPageSize).
|
||||||
|
Find(&repos)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(4, "populateRepoIndexer: %v", err)
|
||||||
|
return
|
||||||
|
} else if len(repos) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
for _, repo := range repos {
|
for _, repo := range repos {
|
||||||
if err = updateRepoIndexer(repo); err != nil {
|
repoIndexerOperationQueue <- repoIndexerOperation{
|
||||||
// only log error, since this should not prevent
|
repo: repo,
|
||||||
// gitea from starting up
|
deleted: false,
|
||||||
log.Error(4, "updateRepoIndexer: repoID=%d, %v", repo.ID, err)
|
}
|
||||||
}
|
maxRepoID = repo.ID - 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
log.Info("Done populating the repo indexer with existing repositories")
|
||||||
}
|
}
|
||||||
|
|
||||||
func updateRepoIndexer(repo *Repository) error {
|
func updateRepoIndexer(repo *Repository) error {
|
||||||
|
|
Loading…
Reference in a new issue