Infrastructure/Cloudreve
0
Fork 0
Code Issues Pull requests Activity

Feat: qiniu upload & callback

Browse source
This commit is contained in:
HFO4 2020-01-15 10:14:15 +08:00
parent 5be7ec98c1
commit e8d21b1e9b
15 changed files with 356 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
go.sum
View file

@ -143,6 +143,7 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/qiniu/api.v7/v7 v7.4.0 h1:9dZMVQifh31QGFLVaHls6akCaS2rlj3du8MnEFd7XjQ=
github.com/qiniu/api.v7/v7 v7.4.0/go.mod h1:VE5oC5rkE1xul0u1S2N0b2Uxq9/6hZzhyqjgK25XDcM=
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438 h1:jnz/4VenymvySjE+Ez511s0pqVzkUOmr1fwCVytNNWk=
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg=

101
middleware/auth.go
View file

@ -5,8 +5,10 @@ import (
"github.com/HFO4/cloudreve/pkg/auth"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/HFO4/cloudreve/pkg/util"
"github.com/gin-contrib/sessions"
"github.com/gin-gonic/gin"
"github.com/qiniu/api.v7/v7/auth/qbox"
"net/http"
)
@ -106,41 +108,45 @@ func WebDAVAuth() gin.HandlerFunc {
}
}
// uploadCallbackCheck 对上传回调请求的 callback key 进行验证，如果成功则返回上传用户
func uploadCallbackCheck(c *gin.Context) (serializer.Response, *model.User) {
// 验证 Callback Key
callbackKey := c.Param("key")
if callbackKey == "" {
return serializer.ParamErr("Callback Key 不能为空", nil), nil
}
callbackSessionRaw, exist := cache.Get("callback_" + callbackKey)
if !exist {
return serializer.ParamErr("回调会话不存在或已过期", nil), nil
}
callbackSession := callbackSessionRaw.(serializer.UploadSession)
c.Set("callbackSession", &callbackSession)
// 清理回调会话
_ = cache.Deletes([]string{callbackKey}, "callback_")
// 查找用户
user, err := model.GetUserByID(callbackSession.UID)
if err != nil {
return serializer.Err(serializer.CodeCheckLogin, "找不到用户", err), nil
}
c.Set("user", &user)
// 检查存储策略是否一致
if user.GetPolicyID() != callbackSession.PolicyID {
return serializer.Err(serializer.CodePolicyNotAllowed, "存储策略已变更，请重新上传", nil), nil
}
return serializer.Response{}, &user
}
// RemoteCallbackAuth 远程回调签名验证
// TODO 测试
func RemoteCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
// 验证 Callback Key
callbackKey := c.Param("key")
if callbackKey == "" {
c.JSON(200, serializer.ParamErr("Callback Key 不能为空", nil))
c.Abort()
return
}
callbackSessionRaw, exist := cache.Get("callback_" + callbackKey)
if !exist {
c.JSON(200, serializer.ParamErr("回调会话不存在或已过期", nil))
c.Abort()
return
}
callbackSession := callbackSessionRaw.(serializer.UploadSession)
c.Set("callbackSession", &callbackSession)
// 清理回调会话
_ = cache.Deletes([]string{callbackKey}, "callback_")
// 查找用户
user, err := model.GetUserByID(callbackSession.UID)
if err != nil {
c.JSON(200, serializer.Err(serializer.CodeCheckLogin, "找不到用户", err))
c.Abort()
return
}
c.Set("user", &user)
// 检查存储策略是否一致
if user.GetPolicyID() != callbackSession.PolicyID {
c.JSON(200, serializer.Err(serializer.CodePolicyNotAllowed, "存储策略已变更，请重新上传", nil))
// 验证key并查找用户
resp, user := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(200, resp)
c.Abort()
return
}
@ -157,3 +163,34 @@ func RemoteCallbackAuth() gin.HandlerFunc {
}
}
// QiniuCallbackAuth 七牛回调签名验证
// TODO 测试
func QiniuCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
// 验证key并查找用户
resp, user := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(401, serializer.QiniuCallbackFailed{Error: resp.Msg})
c.Abort()
return
}
// 验证回调是否来自qiniu
mac := qbox.NewMac(user.Policy.AccessKey, user.Policy.SecretKey)
ok, err := mac.VerifyCallback(c.Request)
if err != nil {
util.Log().Debug("无法验证回调请求，%s", err)
c.JSON(401, serializer.QiniuCallbackFailed{Error: "无法验证回调请求"})
c.Abort()
return
}
if !ok {
c.JSON(401, serializer.QiniuCallbackFailed{Error: "回调签名无效"})
c.Abort()
return
}
c.Next()
}
}

5
models/policy.go
View file

@ -149,6 +149,11 @@ func (policy *Policy) IsDirectlyPreview() bool {
return policy.Type == "local"
}
// IsPathGenerateNeeded 返回此策略是否需要在生成上传凭证时生成存储路径
func (policy *Policy) IsPathGenerateNeeded() bool {
return policy.Type != "remote"
}
// GetUploadURL 获取文件上传服务API地址
func (policy *Policy) GetUploadURL() string {
server, err := url.Parse(policy.Server)

6
pkg/filesystem/filesystem.go
View file

@ -6,6 +6,7 @@ import (
"github.com/HFO4/cloudreve/pkg/auth"
"github.com/HFO4/cloudreve/pkg/conf"
"github.com/HFO4/cloudreve/pkg/filesystem/local"
"github.com/HFO4/cloudreve/pkg/filesystem/qiniu"
"github.com/HFO4/cloudreve/pkg/filesystem/remote"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/request"
@ -159,6 +160,11 @@ func (fs *FileSystem) dispatchHandler() error {
AuthInstance: auth.HMACAuth{[]byte(currentPolicy.SecretKey)},
}
return nil
case "qiniu":
fs.Handler = qiniu.Handler{
Policy: currentPolicy,
}
return nil
default:
return ErrUnknownPolicyType
}

2
pkg/filesystem/hooks.go
View file

@ -224,7 +224,7 @@ func SlaveAfterUpload(ctx context.Context, fs *FileSystem) error {
}
// 发送回调请求
callbackBody := serializer.RemoteUploadCallback{
callbackBody := serializer.UploadCallback{
Name: file.Name,
SourceName: file.SourceName,
PicInfo: file.PicInfo,

87
pkg/filesystem/qiniu/handller.go
View file

@ -2,20 +2,32 @@ package qiniu
import (
"context"
"errors"
"fmt"
"github.com/HFO4/cloudreve/pkg/util"
"io"
"os"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/qiniu/api.v7/v7/auth"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/qiniu/api.v7/v7/storage"
"io"
"net/url"
)
// Handler 本地策略适配器
type Handler struct{}
type Handler struct {
Policy *model.Policy
}
// Get 获取文件
func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, error) {
return nil, errors.New("未实现")
}
// Put 将文件流保存到指定目录
func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
return errors.New("未实现")
// 凭证生成
putPolicy := storage.PutPolicy{
Scope: "cloudrevetest",
@ -44,20 +56,61 @@ func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string,
}
// Delete 删除一个或多个文件，
// 返回已删除的文件，及遇到的最后一个错误
// 返回未删除的文件，及遇到的最后一个错误
func (handler Handler) Delete(ctx context.Context, files []string) ([]string, error) {
deleted := make([]string, 0, len(files))
var retErr error
return []string{}, errors.New("未实现")
}
for _, value := range files {
err := os.Remove(value)
if err == nil {
deleted = append(deleted, value)
} else {
util.Log().Warning("无法删除文件，%s", err)
retErr = err
}
// Thumb 获取文件缩略图
func (handler Handler) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
return nil, errors.New("未实现")
}
// Source 获取外链URL
func (handler Handler) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
return "", errors.New("未实现")
}
// Token 获取上传策略和认证Token
func (handler Handler) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
return deleted, retErr
// 创建上传策略
putPolicy := storage.PutPolicy{
Scope: handler.Policy.BucketName,
Expires: uint64(TTL),
CallbackURL: apiURL.String(),
CallbackBody: `{"name":"$(fname)","source_name":"$(key)","size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
CallbackBodyType: "application/json",
SaveKey: savePath,
ForceSaveKey: true,
FsizeLimit: int64(handler.Policy.MaxSize),
}
// 是否开启了MIMEType限制
if handler.Policy.OptionsSerialized.MimeType != "" {
putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType
}
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
upToken := putPolicy.UploadToken(mac)
return serializer.UploadCredential{
Token: upToken,
}, nil
}

38
pkg/filesystem/template/file.go Normal file
View file

@ -0,0 +1,38 @@
package template
import (
"io"
)
// FileStream 用户传来的文件
type FileStream struct {
File io.ReadCloser
Size uint64
VirtualPath string
Name string
MIMEType string
}
func (file FileStream) Read(p []byte) (n int, err error) {
return file.File.Read(p)
}
func (file FileStream) GetMIMEType() string {
return file.MIMEType
}
func (file FileStream) GetSize() uint64 {
return file.Size
}
func (file FileStream) Close() error {
return file.File.Close()
}
func (file FileStream) GetFileName() string {
return file.Name
}
func (file FileStream) GetVirtualPath() string {
return file.VirtualPath
}

82
pkg/filesystem/template/handller.go Normal file
View file

@ -0,0 +1,82 @@
package template
import (
"context"
"errors"
"fmt"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/qiniu/api.v7/v7/auth"
"github.com/qiniu/api.v7/v7/storage"
"io"
"net/url"
)
// Handler 本地策略适配器
type Handler struct {
Policy *model.Policy
}
// Get 获取文件
func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, error) {
return nil, errors.New("未实现")
}
// Put 将文件流保存到指定目录
func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
return errors.New("未实现")
// 凭证生成
putPolicy := storage.PutPolicy{
Scope: "cloudrevetest",
}
mac := auth.New("YNzTBBpDUq4EEiFV0-vyJCZCJ0LvUEI0_WvxtEXE", "Clm9d9M2CH7pZ8vm049ZlGZStQxrRQVRTjU_T5_0")
upToken := putPolicy.UploadToken(mac)
cfg := storage.Config{}
// 空间对应的机房
cfg.Zone = &storage.ZoneHuadong
formUploader := storage.NewFormUploader(&cfg)
ret := storage.PutRet{}
putExtra := storage.PutExtra{
Params: map[string]string{},
}
defer file.Close()
err := formUploader.Put(ctx, &ret, upToken, dst, file, int64(size), &putExtra)
if err != nil {
fmt.Println(err)
return err
}
fmt.Println(ret.Key, ret.Hash)
return nil
}
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
func (handler Handler) Delete(ctx context.Context, files []string) ([]string, error) {
return []string{}, errors.New("未实现")
}
// Thumb 获取文件缩略图
func (handler Handler) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
return nil, errors.New("未实现")
}
// Source 获取外链URL
func (handler Handler) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
return "", errors.New("未实现")
}
// Token 获取上传策略和认证Token
func (handler Handler) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, errors.New("未实现")
}

6
pkg/filesystem/upload.go
View file

@ -5,6 +5,7 @@ import (
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/filesystem/local"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/HFO4/cloudreve/pkg/util"
"github.com/gin-gonic/gin"
@ -146,6 +147,11 @@ func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint
var err error
// 是否需要预先生成存储路径
if fs.User.Policy.IsPathGenerateNeeded() {
ctx = context.WithValue(ctx, fsctx.SavePathCtx, fs.GenerateSavePath(ctx, local.FileStream{}))
}
// 获取上传凭证
callbackKey := util.RandStringRunes(32)
credential, err := fs.Handler.Token(ctx, int64(credentialTTL), callbackKey)

4
pkg/request/slave.go
View file

@ -11,9 +11,9 @@ import (
)
// RemoteCallback 发送远程存储策略上传回调请求
func RemoteCallback(url string, body serializer.RemoteUploadCallback) error {
func RemoteCallback(url string, body serializer.UploadCallback) error {
callbackBody, err := json.Marshal(struct {
Data serializer.RemoteUploadCallback `json:"data"`
Data serializer.UploadCallback `json:"data"`
}{
Data: body,
})

10
pkg/request/slave_test.go
View file

@ -34,7 +34,7 @@ func TestRemoteCallback(t *testing.T) {
},
})
GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.RemoteUploadCallback{
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
SourceName: "source",
})
asserts.NoError(resp)
@ -59,7 +59,7 @@ func TestRemoteCallback(t *testing.T) {
},
})
GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.RemoteUploadCallback{
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
SourceName: "source",
})
asserts.EqualValues(401, resp.(serializer.AppError).Code)
@ -83,7 +83,7 @@ func TestRemoteCallback(t *testing.T) {
},
})
GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.RemoteUploadCallback{
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
SourceName: "source",
})
asserts.Error(resp)
@ -107,7 +107,7 @@ func TestRemoteCallback(t *testing.T) {
},
})
GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.RemoteUploadCallback{
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
SourceName: "source",
})
asserts.Error(resp)
@ -127,7 +127,7 @@ func TestRemoteCallback(t *testing.T) {
Err: errors.New("error"),
})
GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.RemoteUploadCallback{
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{
SourceName: "source",
})
asserts.Error(resp)

9
pkg/serializer/upload.go
View file

@ -29,14 +29,19 @@ type UploadSession struct {
VirtualPath string
}
// RemoteUploadCallback 远程存储策略上传回调正文
type RemoteUploadCallback struct {
// UploadCallback 上传回调正文
type UploadCallback struct {
Name string `json:"name"`
SourceName string `json:"source_name"`
PicInfo string `json:"pic_info"`
Size uint64 `json:"size"`
}
// QiniuCallbackFailed 七牛存储策略上传回调失败响应
type QiniuCallbackFailed struct {
Error string `json:"error"`
}
func init() {
gob.Register(UploadSession{})
}

18
routers/controllers/callback.go
View file

@ -1,6 +1,7 @@
package controllers
import (
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/HFO4/cloudreve/service/callback"
"github.com/gin-gonic/gin"
)
@ -9,9 +10,24 @@ import (
func RemoteCallback(c *gin.Context) {
var callbackBody callback.RemoteUploadCallbackService
if err := c.ShouldBindJSON(&callbackBody); err == nil {
res := callbackBody.Process(c)
res := callback.ProcessCallback(callbackBody, c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// QiniuCallback 七牛上传回调
func QiniuCallback(c *gin.Context) {
var callbackBody callback.QiniuUploadCallbackService
if err := c.ShouldBindJSON(&callbackBody); err == nil {
res := callback.ProcessCallback(callbackBody, c)
if res.Code != 0 {
c.JSON(401, serializer.QiniuCallbackFailed{Error: res.Msg})
} else {
c.JSON(200, res)
}
} else {
c.JSON(401, ErrorResponse(err))
}
}

8
routers/router.go
View file

@ -126,12 +126,18 @@ func InitMasterRouter() *gin.Engine {
// 回调接口
callback := v3.Group("callback")
{
// 远程上传回调
// 远程策略上传回调
callback.POST(
"remote/:key",
middleware.RemoteCallbackAuth(),
controllers.RemoteCallback,
)
// 七牛策略上传回调
callback.POST(
"qiniu/:key",
middleware.QiniuCallbackAuth(),
controllers.QiniuCallback,
)
}
// 需要登录保护的

50
service/callback/upload.go
View file

@ -10,13 +10,43 @@ import (
"github.com/gin-gonic/gin"
)
// RemoteUploadCallbackService 远程存储上传回调请求服务
type RemoteUploadCallbackService struct {
Data serializer.RemoteUploadCallback `json:"data" binding:"required"`
// CallbackProcessService 上传请求回调正文接口
type CallbackProcessService interface {
GetBody() serializer.UploadCallback
}
// Process 处理远程策略上传结果回调
func (service *RemoteUploadCallbackService) Process(c *gin.Context) serializer.Response {
// RemoteUploadCallbackService 远程存储上传回调请求服务
type RemoteUploadCallbackService struct {
Data serializer.UploadCallback `json:"data" binding:"required"`
}
// GetBody 返回回调正文
func (service RemoteUploadCallbackService) GetBody() serializer.UploadCallback {
return service.Data
}
// QiniuUploadCallbackService 七牛存储上传回调请求服务
type QiniuUploadCallbackService struct {
Name string `json:"name"`
SourceName string `json:"source_name"`
PicInfo string `json:"pic_info"`
Size uint64 `json:"size"`
}
// GetBody 返回回调正文
func (service QiniuUploadCallbackService) GetBody() serializer.UploadCallback {
return serializer.UploadCallback{
Name: service.Name,
SourceName: service.SourceName,
PicInfo: service.PicInfo,
Size: service.Size,
}
}
// ProcessCallback 处理上传结果回调
func ProcessCallback(service CallbackProcessService, c *gin.Context) serializer.Response {
// 获取回调正文
callbackBody := service.GetBody()
// 创建文件系统
fs, err := filesystem.NewFileSystemFromContext(c)
if err != nil {
@ -39,14 +69,14 @@ func (service *RemoteUploadCallbackService) Process(c *gin.Context) serializer.R
// 创建文件头
fileHeader := local.FileStream{
Size: service.Data.Size,
Size: callbackBody.Size,
VirtualPath: callbackSession.VirtualPath,
Name: service.Data.Name,
Name: callbackBody.Name,
}
// 生成上下文
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, fileHeader)
ctx = context.WithValue(ctx, fsctx.SavePathCtx, service.Data.SourceName)
ctx = context.WithValue(ctx, fsctx.SavePathCtx, callbackBody.SourceName)
// 添加钩子
fs.Use("BeforeAddFile", filesystem.HookValidateFile)
@ -60,8 +90,8 @@ func (service *RemoteUploadCallbackService) Process(c *gin.Context) serializer.R
}
// 如果是图片，则更新图片信息
if service.Data.PicInfo != "" {
if err := file.UpdatePicInfo(service.Data.PicInfo); err != nil {
if callbackBody.PicInfo != "" {
if err := file.UpdatePicInfo(callbackBody.PicInfo); err != nil {
util.Log().Debug("无法更新回调文件的图片信息：%s", err)
}
}