2020-04-05 01:20:50 -05:00
// Copyright 2020 The Gitea Authors. All rights reserved.
2022-11-27 13:20:29 -05:00
// SPDX-License-Identifier: MIT
2020-04-05 01:20:50 -05:00
2022-06-12 10:51:54 -05:00
package git
2016-12-25 20:16:37 -05:00
import (
2021-12-09 20:27:50 -05:00
"context"
2022-01-01 04:05:31 -05:00
"fmt"
2022-12-15 15:44:16 -05:00
"time"
2017-12-10 23:37:04 -05:00
2021-09-19 06:49:59 -05:00
"code.gitea.io/gitea/models/db"
2022-06-12 10:51:54 -05:00
"code.gitea.io/gitea/models/perm"
2021-12-09 20:27:50 -05:00
repo_model "code.gitea.io/gitea/models/repo"
2022-06-15 18:24:10 -05:00
"code.gitea.io/gitea/models/unit"
2021-11-24 04:49:20 -05:00
user_model "code.gitea.io/gitea/models/user"
2021-04-08 17:25:57 -05:00
"code.gitea.io/gitea/modules/lfs"
2022-01-01 04:05:31 -05:00
"code.gitea.io/gitea/modules/log"
2022-12-15 15:44:16 -05:00
"code.gitea.io/gitea/modules/setting"
2019-08-15 09:46:21 -05:00
"code.gitea.io/gitea/modules/timeutil"
2022-10-18 00:50:37 -05:00
"code.gitea.io/gitea/modules/util"
2019-10-28 13:31:55 -05:00
"xorm.io/builder"
2016-12-25 20:16:37 -05:00
)
2022-06-12 10:51:54 -05:00
// ErrLFSLockNotExist represents a "LFSLockNotExist" kind of error.
type ErrLFSLockNotExist struct {
ID int64
RepoID int64
Path string
}
// IsErrLFSLockNotExist checks if an error is a ErrLFSLockNotExist.
func IsErrLFSLockNotExist ( err error ) bool {
_ , ok := err . ( ErrLFSLockNotExist )
return ok
}
func ( err ErrLFSLockNotExist ) Error ( ) string {
return fmt . Sprintf ( "lfs lock does not exist [id: %d, rid: %d, path: %s]" , err . ID , err . RepoID , err . Path )
}
2022-10-18 00:50:37 -05:00
func ( err ErrLFSLockNotExist ) Unwrap ( ) error {
return util . ErrNotExist
}
2022-06-12 10:51:54 -05:00
// ErrLFSUnauthorizedAction represents a "LFSUnauthorizedAction" kind of error.
type ErrLFSUnauthorizedAction struct {
RepoID int64
UserName string
Mode perm . AccessMode
}
// IsErrLFSUnauthorizedAction checks if an error is a ErrLFSUnauthorizedAction.
func IsErrLFSUnauthorizedAction ( err error ) bool {
_ , ok := err . ( ErrLFSUnauthorizedAction )
return ok
}
func ( err ErrLFSUnauthorizedAction ) Error ( ) string {
if err . Mode == perm . AccessModeWrite {
return fmt . Sprintf ( "User %s doesn't have write access for lfs lock [rid: %d]" , err . UserName , err . RepoID )
}
return fmt . Sprintf ( "User %s doesn't have read access for lfs lock [rid: %d]" , err . UserName , err . RepoID )
}
2022-10-18 00:50:37 -05:00
func ( err ErrLFSUnauthorizedAction ) Unwrap ( ) error {
return util . ErrPermissionDenied
}
2022-06-12 10:51:54 -05:00
// ErrLFSLockAlreadyExist represents a "LFSLockAlreadyExist" kind of error.
type ErrLFSLockAlreadyExist struct {
RepoID int64
Path string
}
// IsErrLFSLockAlreadyExist checks if an error is a ErrLFSLockAlreadyExist.
func IsErrLFSLockAlreadyExist ( err error ) bool {
_ , ok := err . ( ErrLFSLockAlreadyExist )
return ok
}
func ( err ErrLFSLockAlreadyExist ) Error ( ) string {
return fmt . Sprintf ( "lfs lock already exists [rid: %d, path: %s]" , err . RepoID , err . Path )
}
2022-10-18 00:50:37 -05:00
func ( err ErrLFSLockAlreadyExist ) Unwrap ( ) error {
return util . ErrAlreadyExist
}
2022-06-12 10:51:54 -05:00
// ErrLFSFileLocked represents a "LFSFileLocked" kind of error.
type ErrLFSFileLocked struct {
RepoID int64
Path string
UserName string
}
// IsErrLFSFileLocked checks if an error is a ErrLFSFileLocked.
func IsErrLFSFileLocked ( err error ) bool {
_ , ok := err . ( ErrLFSFileLocked )
return ok
}
func ( err ErrLFSFileLocked ) Error ( ) string {
return fmt . Sprintf ( "File is lfs locked [repo: %d, locked by: %s, path: %s]" , err . RepoID , err . UserName , err . Path )
}
2022-10-18 00:50:37 -05:00
func ( err ErrLFSFileLocked ) Unwrap ( ) error {
return util . ErrPermissionDenied
}
2016-12-25 20:16:37 -05:00
// LFSMetaObject stores metadata for LFS tracked files.
type LFSMetaObject struct {
2021-04-08 17:25:57 -05:00
ID int64 ` xorm:"pk autoincr" `
lfs . Pointer ` xorm:"extends" `
2019-08-15 09:46:21 -05:00
RepositoryID int64 ` xorm:"UNIQUE(s) INDEX NOT NULL" `
Existing bool ` xorm:"-" `
CreatedUnix timeutil . TimeStamp ` xorm:"created" `
2016-12-25 20:16:37 -05:00
}
2021-09-19 06:49:59 -05:00
func init ( ) {
db . RegisterModel ( new ( LFSMetaObject ) )
}
2016-12-25 20:16:37 -05:00
// LFSTokenResponse defines the JSON structure in which the JWT token is stored.
// This structure is fetched via SSH and passed by the Git LFS client to the server
// endpoint for authorization.
type LFSTokenResponse struct {
Header map [ string ] string ` json:"header" `
Href string ` json:"href" `
}
2021-03-14 13:52:12 -05:00
// ErrLFSObjectNotExist is returned from lfs models functions in order
// to differentiate between database and missing object errors.
2022-10-18 00:50:37 -05:00
var ErrLFSObjectNotExist = db . ErrNotExist { Resource : "LFS Meta object" }
2016-12-25 20:16:37 -05:00
// NewLFSMetaObject stores a given populated LFSMetaObject structure in the database
// if it is not already present.
2023-01-08 22:50:54 -05:00
func NewLFSMetaObject ( ctx context . Context , m * LFSMetaObject ) ( * LFSMetaObject , error ) {
2016-12-25 20:16:37 -05:00
var err error
2023-01-08 22:50:54 -05:00
ctx , committer , err := db . TxContext ( ctx )
2021-11-21 10:41:00 -05:00
if err != nil {
2019-01-23 03:56:51 -05:00
return nil , err
}
2021-11-21 10:41:00 -05:00
defer committer . Close ( )
2019-01-23 03:56:51 -05:00
2021-11-21 10:41:00 -05:00
has , err := db . GetByBean ( ctx , m )
2016-12-25 20:16:37 -05:00
if err != nil {
return nil , err
}
if has {
m . Existing = true
2021-11-21 10:41:00 -05:00
return m , committer . Commit ( )
2016-12-25 20:16:37 -05:00
}
2021-11-21 10:41:00 -05:00
if err = db . Insert ( ctx , m ) ; err != nil {
2016-12-25 20:16:37 -05:00
return nil , err
}
2021-11-21 10:41:00 -05:00
return m , committer . Commit ( )
2016-12-25 20:16:37 -05:00
}
// GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID.
// It may return ErrLFSObjectNotExist or a database error. If the error is nil,
// the returned pointer is a valid LFSMetaObject.
2023-01-08 22:50:54 -05:00
func GetLFSMetaObjectByOid ( ctx context . Context , repoID int64 , oid string ) ( * LFSMetaObject , error ) {
2016-12-25 20:16:37 -05:00
if len ( oid ) == 0 {
return nil , ErrLFSObjectNotExist
}
2021-12-09 20:27:50 -05:00
m := & LFSMetaObject { Pointer : lfs . Pointer { Oid : oid } , RepositoryID : repoID }
2023-01-08 22:50:54 -05:00
has , err := db . GetEngine ( ctx ) . Get ( m )
2016-12-25 20:16:37 -05:00
if err != nil {
return nil , err
} else if ! has {
return nil , ErrLFSObjectNotExist
}
return m , nil
}
// RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID.
// It may return ErrLFSObjectNotExist or a database error.
2023-01-08 22:50:54 -05:00
func RemoveLFSMetaObjectByOid ( ctx context . Context , repoID int64 , oid string ) ( int64 , error ) {
return RemoveLFSMetaObjectByOidFn ( ctx , repoID , oid , nil )
2022-12-15 15:44:16 -05:00
}
// RemoveLFSMetaObjectByOidFn removes a LFSMetaObject entry from database by its OID.
// It may return ErrLFSObjectNotExist or a database error. It will run Fn with the current count within the transaction
2023-01-08 22:50:54 -05:00
func RemoveLFSMetaObjectByOidFn ( ctx context . Context , repoID int64 , oid string , fn func ( count int64 ) error ) ( int64 , error ) {
2016-12-25 20:16:37 -05:00
if len ( oid ) == 0 {
2019-10-28 13:31:55 -05:00
return 0 , ErrLFSObjectNotExist
2016-12-25 20:16:37 -05:00
}
2023-01-08 22:50:54 -05:00
ctx , committer , err := db . TxContext ( ctx )
2021-11-21 10:41:00 -05:00
if err != nil {
return 0 , err
2016-12-25 20:16:37 -05:00
}
2021-11-21 10:41:00 -05:00
defer committer . Close ( )
2016-12-25 20:16:37 -05:00
2021-12-09 20:27:50 -05:00
m := & LFSMetaObject { Pointer : lfs . Pointer { Oid : oid } , RepositoryID : repoID }
2021-11-21 10:41:00 -05:00
if _ , err := db . DeleteByBean ( ctx , m ) ; err != nil {
2019-10-28 13:31:55 -05:00
return - 1 , err
}
2021-11-21 10:41:00 -05:00
count , err := db . CountByBean ( ctx , & LFSMetaObject { Pointer : lfs . Pointer { Oid : oid } } )
2019-10-28 13:31:55 -05:00
if err != nil {
return count , err
}
2022-12-15 15:44:16 -05:00
if fn != nil {
if err := fn ( count ) ; err != nil {
return count , err
}
}
2021-11-21 10:41:00 -05:00
return count , committer . Commit ( )
2019-10-28 13:31:55 -05:00
}
// GetLFSMetaObjects returns all LFSMetaObjects associated with a repository
2023-01-08 22:50:54 -05:00
func GetLFSMetaObjects ( ctx context . Context , repoID int64 , page , pageSize int ) ( [ ] * LFSMetaObject , error ) {
sess := db . GetEngine ( ctx )
2019-10-28 13:31:55 -05:00
if page >= 0 && pageSize > 0 {
start := 0
if page > 0 {
start = ( page - 1 ) * pageSize
}
sess . Limit ( pageSize , start )
}
lfsObjects := make ( [ ] * LFSMetaObject , 0 , pageSize )
2021-12-09 20:27:50 -05:00
return lfsObjects , sess . Find ( & lfsObjects , & LFSMetaObject { RepositoryID : repoID } )
2019-10-28 13:31:55 -05:00
}
// CountLFSMetaObjects returns a count of all LFSMetaObjects associated with a repository
2023-01-08 22:50:54 -05:00
func CountLFSMetaObjects ( ctx context . Context , repoID int64 ) ( int64 , error ) {
return db . GetEngine ( ctx ) . Count ( & LFSMetaObject { RepositoryID : repoID } )
2019-10-28 13:31:55 -05:00
}
// LFSObjectAccessible checks if a provided Oid is accessible to the user
2023-01-08 22:50:54 -05:00
func LFSObjectAccessible ( ctx context . Context , user * user_model . User , oid string ) ( bool , error ) {
2019-10-28 13:31:55 -05:00
if user . IsAdmin {
2023-01-08 22:50:54 -05:00
count , err := db . GetEngine ( ctx ) . Count ( & LFSMetaObject { Pointer : lfs . Pointer { Oid : oid } } )
2021-04-09 02:40:34 -05:00
return count > 0 , err
2019-10-28 13:31:55 -05:00
}
2022-06-15 18:24:10 -05:00
cond := repo_model . AccessibleRepositoryCondition ( user , unit . TypeInvalid )
2023-01-08 22:50:54 -05:00
count , err := db . GetEngine ( ctx ) . Where ( cond ) . Join ( "INNER" , "repository" , "`lfs_meta_object`.repository_id = `repository`.id" ) . Count ( & LFSMetaObject { Pointer : lfs . Pointer { Oid : oid } } )
2021-04-09 02:40:34 -05:00
return count > 0 , err
2019-10-28 13:31:55 -05:00
}
2022-11-15 03:08:59 -05:00
// ExistsLFSObject checks if a provided Oid exists within the DB
func ExistsLFSObject ( ctx context . Context , oid string ) ( bool , error ) {
return db . GetEngine ( ctx ) . Exist ( & LFSMetaObject { Pointer : lfs . Pointer { Oid : oid } } )
2022-01-01 04:05:31 -05:00
}
2019-10-28 13:31:55 -05:00
// LFSAutoAssociate auto associates accessible LFSMetaObjects
2023-01-08 22:50:54 -05:00
func LFSAutoAssociate ( ctx context . Context , metas [ ] * LFSMetaObject , user * user_model . User , repoID int64 ) error {
ctx , committer , err := db . TxContext ( ctx )
2021-11-21 10:41:00 -05:00
if err != nil {
2019-10-28 13:31:55 -05:00
return err
}
2021-11-21 10:41:00 -05:00
defer committer . Close ( )
sess := db . GetEngine ( ctx )
2019-10-28 13:31:55 -05:00
oids := make ( [ ] interface { } , len ( metas ) )
oidMap := make ( map [ string ] * LFSMetaObject , len ( metas ) )
for i , meta := range metas {
oids [ i ] = meta . Oid
oidMap [ meta . Oid ] = meta
}
if ! user . IsAdmin {
2022-01-01 04:05:31 -05:00
newMetas := make ( [ ] * LFSMetaObject , 0 , len ( metas ) )
cond := builder . In (
"`lfs_meta_object`.repository_id" ,
2022-06-15 18:24:10 -05:00
builder . Select ( "`repository`.id" ) . From ( "repository" ) . Where ( repo_model . AccessibleRepositoryCondition ( user , unit . TypeInvalid ) ) ,
2022-01-01 04:05:31 -05:00
)
err = sess . Cols ( "oid" ) . Where ( cond ) . In ( "oid" , oids ... ) . GroupBy ( "oid" ) . Find ( & newMetas )
if err != nil {
return err
}
if len ( newMetas ) != len ( oidMap ) {
return fmt . Errorf ( "unable collect all LFS objects from database, expected %d, actually %d" , len ( oidMap ) , len ( newMetas ) )
}
for i := range newMetas {
newMetas [ i ] . Size = oidMap [ newMetas [ i ] . Oid ] . Size
newMetas [ i ] . RepositoryID = repoID
}
if err = db . Insert ( ctx , newMetas ) ; err != nil {
return err
}
} else {
// admin can associate any LFS object to any repository, and we do not care about errors (eg: duplicated unique key),
// even if error occurs, it won't hurt users and won't make things worse
for i := range metas {
2022-03-10 05:09:48 -05:00
p := lfs . Pointer { Oid : metas [ i ] . Oid , Size : metas [ i ] . Size }
2022-01-01 04:05:31 -05:00
_ , err = sess . Insert ( & LFSMetaObject {
2022-03-10 05:09:48 -05:00
Pointer : p ,
2022-01-01 04:05:31 -05:00
RepositoryID : repoID ,
} )
if err != nil {
2022-03-10 05:09:48 -05:00
log . Warn ( "failed to insert LFS meta object %-v for repo_id: %d into database, err=%v" , p , repoID , err )
2022-01-01 04:05:31 -05:00
}
}
2016-12-25 20:16:37 -05:00
}
2021-11-21 10:41:00 -05:00
return committer . Commit ( )
2016-12-25 20:16:37 -05:00
}
2020-09-08 10:45:10 -05:00
2021-12-09 20:27:50 -05:00
// CopyLFS copies LFS data from one repo to another
func CopyLFS ( ctx context . Context , newRepo , oldRepo * repo_model . Repository ) error {
var lfsObjects [ ] * LFSMetaObject
if err := db . GetEngine ( ctx ) . Where ( "repository_id=?" , oldRepo . ID ) . Find ( & lfsObjects ) ; err != nil {
return err
}
for _ , v := range lfsObjects {
v . ID = 0
v . RepositoryID = newRepo . ID
2022-06-12 10:51:54 -05:00
if err := db . Insert ( ctx , v ) ; err != nil {
2021-12-09 20:27:50 -05:00
return err
}
}
return nil
}
2022-06-06 03:01:49 -05:00
// GetRepoLFSSize return a repository's lfs files size
func GetRepoLFSSize ( ctx context . Context , repoID int64 ) ( int64 , error ) {
lfsSize , err := db . GetEngine ( ctx ) . Where ( "repository_id = ?" , repoID ) . SumInt ( new ( LFSMetaObject ) , "size" )
if err != nil {
2022-10-24 14:29:17 -05:00
return 0 , fmt . Errorf ( "updateSize: GetLFSMetaObjects: %w" , err )
2022-06-06 03:01:49 -05:00
}
return lfsSize , nil
}
2022-12-15 15:44:16 -05:00
type IterateLFSMetaObjectsForRepoOptions struct {
OlderThan time . Time
}
// IterateLFSMetaObjectsForRepo provides a iterator for LFSMetaObjects per Repo
func IterateLFSMetaObjectsForRepo ( ctx context . Context , repoID int64 , f func ( context . Context , * LFSMetaObject , int64 ) error , opts * IterateLFSMetaObjectsForRepoOptions ) error {
var start int
batchSize := setting . Database . IterateBufferSize
engine := db . GetEngine ( ctx )
type CountLFSMetaObject struct {
Count int64
LFSMetaObject
}
for {
beans := make ( [ ] * CountLFSMetaObject , 0 , batchSize )
// SELECT `lfs_meta_object`.*, COUNT(`l1`.id) as `count` FROM lfs_meta_object INNER JOIN lfs_meta_object AS l1 ON l1.oid = lfs_meta_object.oid WHERE lfs_meta_object.repository_id = ? GROUP BY lfs_meta_object.id
sess := engine . Select ( "`lfs_meta_object`.*, COUNT(`l1`.oid) AS `count`" ) .
Join ( "INNER" , "`lfs_meta_object` AS l1" , "`lfs_meta_object`.oid = `l1`.oid" ) .
Where ( "`lfs_meta_object`.repository_id = ?" , repoID )
if ! opts . OlderThan . IsZero ( ) {
sess . And ( "`lfs_meta_object`.created_unix < ?" , opts . OlderThan )
}
sess . GroupBy ( "`lfs_meta_object`.id" )
if err := sess . Limit ( batchSize , start ) . Find ( & beans ) ; err != nil {
return err
}
if len ( beans ) == 0 {
return nil
}
start += len ( beans )
for _ , bean := range beans {
if err := f ( ctx , & bean . LFSMetaObject , bean . Count ) ; err != nil {
return err
}
}
}
}