Infrastructure/Cloudreve
0
Fork 0
Code Issues Pull requests Activity

Feat: create upload session and pre-upload check

Browse source
This commit is contained in:
HFO4 2022-02-10 19:30:08 +08:00
parent 855c9d92c4
commit de9c41082c
18 changed files with 82 additions and 135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
pkg/filesystem/driver/cos/handler.go
View file

@ -324,7 +324,7 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
@ -333,7 +333,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + key)
apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI).String()
// 上传策略
@ -346,7 +346,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
map[string]string{"bucket": handler.Policy.BucketName},
map[string]string{"$key": savePath},
map[string]string{"x-cos-meta-callback": apiURL},
map[string]string{"x-cos-meta-key": key},
map[string]string{"x-cos-meta-key": uploadSession.Key},
map[string]string{"q-sign-algorithm": "sha1"},
map[string]string{"q-ak": handler.Policy.AccessKey},
map[string]string{"q-sign-time": keyTime},
@ -361,7 +361,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
res, err := handler.getUploadCredential(ctx, postPolicy, keyTime)
if err == nil {
res.Callback = apiURL
res.Key = key
res.Key = uploadSession.Key
}
return res, err

4
pkg/filesystem/driver/handler.go
View file

@ -29,8 +29,8 @@ type Handler interface {
// isDownload - 是否直接下载
Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool, speed int) (string, error)
// Token 获取有效期为ttl的上传凭证和签名，同时回调会话有效期为sessionTTL
Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error)
// Token 获取有效期为ttl的上传凭证和签名
Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error)
// List 递归列取远程端path路径下文件、目录，不包含path本身，
// 返回的对象路径以path作为起始根目录.

19
pkg/filesystem/driver/local/handler.go
View file

@ -79,22 +79,9 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
return nil, err
}
// 开启一个协程，用于请求结束后关闭reader
// go closeReader(ctx, file)
return file, nil
}
// closeReader 用于在请求结束后关闭reader
// TODO 让业务代码自己关闭
func closeReader(ctx context.Context, closer io.Closer) {
select {
case <-ctx.Done():
_ = closer.Close()
}
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
defer file.Close()
@ -227,6 +214,8 @@ func (handler Driver) Source(
}
// Token 获取上传策略和认证Token，本地策略直接返回空值
func (handler Driver) Token(ctx context.Context, ttl int64, key string) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, nil
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
return serializer.UploadCredential{
SessionID: uploadSession.Key,
}, nil
}

6
pkg/filesystem/driver/onedrive/handler.go
View file

@ -223,7 +223,7 @@ func (handler Driver) replaceSourceHost(origin string) (string, error) {
}
// Token 获取上传会话URL
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径和文件大小
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
@ -242,7 +242,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/onedrive/finish/" + key)
apiBaseURI, _ := url.Parse("/api/v3/callback/onedrive/finish/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
uploadURL, err := handler.Client.CreateUploadSession(ctx, savePath, WithConflictBehavior("fail"))
@ -251,7 +251,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
}
// 监控回调及上传
go handler.Client.MonitorUpload(uploadURL, key, savePath, fileSize, TTL)
go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, savePath, fileSize, ttl)
return serializer.UploadCredential{
Policy: uploadURL,

4
pkg/filesystem/driver/oss/handler.go
View file

@ -397,7 +397,7 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
@ -406,7 +406,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + key)
apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 回调策略

4
pkg/filesystem/driver/qiniu/handler.go
View file

@ -274,10 +274,10 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64)
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + key)
apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 读取上下文中生成的存储路径

4
pkg/filesystem/driver/remote/handler.go
View file

@ -305,10 +305,10 @@ func (handler Driver) Source(
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/remote/" + key)
apiBaseURI, _ := url.Parse("/api/v3/callback/remote/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 生成上传策略

4
pkg/filesystem/driver/s3/handler.go
View file

@ -324,7 +324,7 @@ func (handler Driver) Source(
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径和文件大小
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
@ -334,7 +334,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/s3/" + key)
apiBaseURI, _ := url.Parse("/api/v3/callback/s3/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 上传策略

2
pkg/filesystem/driver/shadow/masterinslave/handler.go
View file

@ -47,7 +47,7 @@ func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64
return "", ErrNotImplemented
}
func (d *Driver) Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error) {
func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, ErrNotImplemented
}

2
pkg/filesystem/driver/shadow/slaveinmaster/handler.go
View file

@ -112,7 +112,7 @@ func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64
return "", ErrNotImplemented
}
func (d *Driver) Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error) {
func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, ErrNotImplemented
}

55
pkg/filesystem/driver/template/handler.go
View file

@ -1,55 +0,0 @@
package template
import (
"context"
"errors"
"io"
"net/url"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
)
// Driver 适配器模板
type Driver struct {
Policy *model.Policy
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
return nil, errors.New("未实现")
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
return errors.New("未实现")
}
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
return []string{}, errors.New("未实现")
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
return nil, errors.New("未实现")
}
// Source 获取外链URL
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
return "", errors.New("未实现")
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, errors.New("未实现")
}

4
pkg/filesystem/driver/upyun/handler.go
View file

@ -311,7 +311,7 @@ func (handler Driver) signURL(ctx context.Context, path *url.URL, TTL int64) (st
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, TTL int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径和文件大小
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
@ -326,7 +326,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + key)
apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 上传策略

55
pkg/filesystem/upload.go
View file

@ -14,6 +14,7 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
"github.com/gofrs/uuid"
)
/* ================
@ -146,32 +147,47 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file FileHe
}
}
// GetUploadToken 生成新的上传凭证
func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint64, name string) (*serializer.UploadCredential, error) {
// CreateUploadSession 创建上传会话
func (fs *FileSystem) CreateUploadSession(ctx context.Context, path string, size uint64, name string) (*serializer.UploadCredential, error) {
// 获取相关有效期设置
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
callBackSessionTTL := model.GetIntSetting("upload_session_timeout", 86400)
var err error
// 检查文件大小
if fs.Policy.MaxSize != 0 {
if size > fs.Policy.MaxSize {
return nil, ErrFileSizeTooBig
}
// 进行文件上传预检查
// 创建上下文环境
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, local.FileStream{
Size: size,
Name: name,
})
// 检查上传请求合法性
if err := HookValidateFile(ctx, fs); err != nil {
return nil, err
}
// 是否需要预先生成存储路径
var savePath string
if fs.Policy.IsPathGenerateNeeded() {
savePath = fs.GenerateSavePath(ctx, local.FileStream{Name: name, VirtualPath: path})
ctx = context.WithValue(ctx, fsctx.SavePathCtx, savePath)
if err := HookValidateCapacityWithoutIncrease(ctx, fs); err != nil {
return nil, err
}
// 生成存储路径
savePath := fs.GenerateSavePath(ctx, local.FileStream{Name: name, VirtualPath: path})
callbackKey := uuid.Must(uuid.NewV4()).String()
uploadSession := &serializer.UploadSession{
Key: callbackKey,
UID: fs.User.ID,
PolicyID: fs.Policy.ID,
VirtualPath: path,
Name: name,
Size: size,
SavePath: savePath,
}
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, size)
// 获取上传凭证
callbackKey := util.RandStringRunes(32)
credential, err := fs.Handler.Token(ctx, int64(credentialTTL), callbackKey)
credential, err := fs.Handler.Token(ctx, int64(credentialTTL), uploadSession)
if err != nil {
return nil, serializer.NewError(serializer.CodeEncryptError, "无法获取上传凭证", err)
}
@ -179,14 +195,7 @@ func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint
// 创建回调会话
err = cache.Set(
"callback_"+callbackKey,
serializer.UploadSession{
Key: callbackKey,
UID: fs.User.ID,
VirtualPath: path,
Name: name,
Size: size,
SavePath: savePath,
},
uploadSession,
callBackSessionTTL,
)
if err != nil {

8
pkg/filesystem/upload_test.go
View file

@ -57,8 +57,8 @@ func (m FileHeaderMock) Source(ctx context.Context, path string, url url.URL, ex
return args.Get(0).(string), args.Error(1)
}
func (m FileHeaderMock) Token(ctx context.Context, expires int64, key string) (serializer.UploadCredential, error) {
args := m.Called(ctx, expires, key)
func (m FileHeaderMock) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession) (serializer.UploadCredential, error) {
args := m.Called(ctx, ttl, uploadSession)
return args.Get(0).(serializer.UploadCredential), args.Error(1)
}
@ -189,7 +189,7 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
testHandler := new(FileHeaderMock)
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{Token: "test"}, nil)
fs.Handler = testHandler
res, err := fs.GetUploadToken(ctx, "/", 10, "123")
res, err := fs.CreateUploadSession(ctx, "/", 10, "123")
testHandler.AssertExpectations(t)
asserts.NoError(err)
asserts.Equal("test", res.Token)
@ -204,7 +204,7 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
testHandler := new(FileHeaderMock)
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{}, errors.New("error"))
fs.Handler = testHandler
_, err := fs.GetUploadToken(ctx, "/", 10, "123")
_, err := fs.CreateUploadSession(ctx, "/", 10, "123")
testHandler.AssertExpectations(t)
asserts.Error(err)
}

2
pkg/serializer/upload.go
View file

@ -18,6 +18,8 @@ type UploadPolicy struct {
// UploadCredential 返回给客户端的上传凭证
type UploadCredential struct {
SessionID string `json:"sessionID"`
Token string `json:"token"`
Policy string `json:"policy"`
Path string `json:"path"` // 存储路径

8
routers/controllers/file.go
View file

@ -354,15 +354,15 @@ func FileUploadStream(c *gin.Context) {
})
}
// GetUploadCredential 获取上传凭证
// GetUploadCredential 创建上传会话
func GetUploadCredential(c *gin.Context) {
// 创建上下文
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var service explorer.UploadCredentialService
if err := c.ShouldBindQuery(&service); err == nil {
res := service.Get(ctx, c)
var service explorer.UploadSessionService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.Create(ctx, c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))

4
routers/router.go
View file

@ -506,8 +506,8 @@ func InitMasterRouter() *gin.Engine {
{
// 文件上传
file.POST("upload", controllers.FileUploadStream)
// 获取上传凭证
file.GET("upload/credential", controllers.GetUploadCredential)
// 创建上传会话
file.PUT("upload/session", controllers.GetUploadCredential)
// 更新文件
file.PUT("update/:id", controllers.PutContent)
// 创建空白文件

24
service/explorer/upload.go
View file

@ -4,29 +4,31 @@ import (
"context"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
)
// UploadCredentialService 获取上传凭证服务
type UploadCredentialService struct {
Path string `form:"path" binding:"required"`
Size uint64 `form:"size" binding:"min=0"`
Name string `form:"name"`
Type string `form:"type"`
// UploadSessionService 获取上传凭证服务
type UploadSessionService struct {
Path string `json:"path" binding:"required"`
Size uint64 `json:"size" binding:"min=0"`
Name string `json:"name" binding:"required"`
PolicyID uint `json:"policy_id" binding:"required"`
}
// Get 获取新的上传凭证
func (service *UploadCredentialService) Get(ctx context.Context, c *gin.Context) serializer.Response {
// Create 创建新的上传会话
func (service *UploadSessionService) Create(ctx context.Context, c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewFileSystemFromContext(c)
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
credential, err := fs.GetUploadToken(ctx, service.Path, service.Size, service.Name)
if fs.Policy.ID != service.PolicyID {
return serializer.Err(serializer.CodePolicyNotAllowed, "存储策略发生变化，请刷新文件列表并重新添加此任务", nil)
}
credential, err := fs.CreateUploadSession(ctx, service.Path, service.Size, service.Name)
if err != nil {
return serializer.Err(serializer.CodeNotSet, err.Error(), err)
}