Infrastructure/Cloudreve
0
Fork 0
Code Issues Pull requests Activity

Feat: COS credential / upload callback

Browse source
This commit is contained in:
HFO4 2020-01-23 12:38:32 +08:00
parent e894450d5e
commit 8d437a451c
11 changed files with 282 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
go.mod
View file

@ -26,6 +26,7 @@ require (
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/stretchr/testify v1.4.0
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac
github.com/upyun/go-sdk v2.1.0+incompatible
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
gopkg.in/go-playground/validator.v8 v8.18.2

8
go.sum
View file

@ -4,6 +4,7 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI=
@ -73,8 +74,11 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLmwsOARdV86pfH3g95wXmE=
github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
@ -126,6 +130,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2 h1:daZqE/T/yEoKIQNd3rwNeLsiS0VpZFfJulR0t/rtgAE=
github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2/go.mod h1:wAQCKEc5bDujxKRmbT6/vTnTt5CjStQ8bRfPWUuz/iY=
github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
@ -165,6 +171,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ=
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=

16
middleware/auth.go
View file

@ -293,3 +293,19 @@ func OneDriveCallbackAuth() gin.HandlerFunc {
c.Next()
}
}
// COSCallbackAuth 腾讯云COS回调签名验证
// TODO 解耦 测试
func COSCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
// 验证key并查找用户
resp, _ := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(401, serializer.QiniuCallbackFailed{Error: resp.Msg})
c.Abort()
return
}
c.Next()
}
}

4
models/policy.go
View file

@ -145,7 +145,7 @@ func (policy Policy) getOriginNameRule(origin string) string {
return "$(fname)"
case "local", "remote":
return origin
case "oss":
case "oss", "cos":
// OSS会将${filename}自动替换为原始文件名
return "${filename}"
case "upyun":
@ -201,7 +201,7 @@ func (policy *Policy) GetUploadURL() string {
controller, _ = url.Parse("/api/v3/file/upload")
case "remote":
controller, _ = url.Parse("/api/v3/slave/upload")
case "oss":
case "oss", "cos":
return policy.BaseURL
case "upyun":
return "http://v0.api.upyun.com/" + policy.BucketName

171
pkg/filesystem/driver/cos/handller.go Normal file
View file

@ -0,0 +1,171 @@
package cos
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/serializer"
cossdk "github.com/tencentyun/cos-go-sdk-v5"
"io"
"net/url"
"time"
)
// UploadPolicy 腾讯云COS上传策略
type UploadPolicy struct {
Expiration string `json:"expiration"`
Conditions []interface{} `json:"conditions"`
}
// MetaData 文件元信息
type MetaData struct {
Size uint64
CallbackKey string
CallbackURL string
}
// Driver 腾讯云COS适配器模板
type Driver struct {
Policy *model.Policy
Client *cossdk.Client
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
return nil, errors.New("未实现")
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
return errors.New("未实现")
}
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
return []string{}, errors.New("未实现")
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
return nil, errors.New("未实现")
}
// Source 获取外链URL
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
return "", errors.New("未实现")
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI).String()
// 上传策略
startTime := time.Now()
endTime := startTime.Add(time.Duration(TTL) * time.Second)
keyTime := fmt.Sprintf("%d;%d", startTime.Unix(), endTime.Unix())
postPolicy := UploadPolicy{
Expiration: endTime.UTC().Format(time.RFC3339),
Conditions: []interface{}{
map[string]string{"bucket": handler.Policy.BucketName},
map[string]string{"$key": savePath},
map[string]string{"x-cos-meta-callback": apiURL},
map[string]string{"x-cos-meta-key": key},
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize},
map[string]string{"q-sign-algorithm": "sha1"},
map[string]string{"q-ak": handler.Policy.AccessKey},
map[string]string{"q-sign-time": keyTime},
},
}
res, err := handler.getUploadCredential(ctx, postPolicy, keyTime)
if err == nil {
res.Callback = apiURL
res.Key = key
}
return res, err
}
// Meta 获取文件信息
func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
res, err := handler.Client.Object.Head(ctx, path, &cossdk.ObjectHeadOptions{})
if err != nil {
return nil, err
}
return &MetaData{
Size: uint64(res.ContentLength),
CallbackKey: res.Header.Get("x-cos-meta-key"),
CallbackURL: res.Header.Get("x-cos-meta-callback"),
}, nil
}
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, keyTime string) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
// 编码上传策略
policyJSON, err := json.Marshal(policy)
if err != nil {
return serializer.UploadCredential{}, err
}
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
// 签名上传策略
hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
_, err = io.WriteString(hmacSign, keyTime)
if err != nil {
return serializer.UploadCredential{}, err
}
signKey := fmt.Sprintf("%x", hmacSign.Sum(nil))
sha1Sign := sha1.New()
_, err = sha1Sign.Write(policyJSON)
if err != nil {
return serializer.UploadCredential{}, err
}
stringToSign := fmt.Sprintf("%x", sha1Sign.Sum(nil))
// 最终签名
hmacFinalSign := hmac.New(sha1.New, []byte(signKey))
_, err = hmacFinalSign.Write([]byte(stringToSign))
if err != nil {
return serializer.UploadCredential{}, err
}
signature := hmacFinalSign.Sum(nil)
return serializer.UploadCredential{
Policy: policyEncoded,
Path: savePath,
AccessKey: handler.Policy.AccessKey,
Token: fmt.Sprintf("%x", signature),
KeyTime: keyTime,
}, nil
}

16
pkg/filesystem/filesystem.go
View file

@ -6,6 +6,7 @@ import (
"github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/auth"
"github.com/HFO4/cloudreve/pkg/conf"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/cos"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/onedrive"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/oss"
@ -16,7 +17,9 @@ import (
"github.com/HFO4/cloudreve/pkg/request"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/gin-gonic/gin"
cossdk "github.com/tencentyun/cos-go-sdk-v5"
"io"
"net/http"
"net/url"
"sync"
)
@ -191,6 +194,19 @@ func (fs *FileSystem) DispatchHandler() error {
HTTPClient: request.HTTPClient{},
}
return err
case "cos":
u, _ := url.Parse(currentPolicy.Server)
b := &cossdk.BaseURL{BucketURL: u}
fs.Handler = cos.Driver{
Policy: currentPolicy,
Client: cossdk.NewClient(b, &http.Client{
Transport: &cossdk.AuthorizationTransport{
SecretID: currentPolicy.AccessKey,
SecretKey: currentPolicy.SecretKey,
},
}),
}
return nil
default:
return ErrUnknownPolicyType
}

1
pkg/filesystem/upload.go
View file

@ -165,6 +165,7 @@ func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint
err = cache.Set(
"callback_"+callbackKey,
serializer.UploadSession{
Key: callbackKey,
UID: fs.User.ID,
PolicyID: fs.User.GetPolicyID(),
VirtualPath: path,

6
pkg/serializer/upload.go
View file

@ -20,12 +20,16 @@ type UploadPolicy struct {
type UploadCredential struct {
Token string `json:"token"`
Policy string `json:"policy"`
Path string `json:"path"`
Path string `json:"path"` // 存储路径
AccessKey string `json:"ak"`
KeyTime string `json:"key_time,omitempty"` // COS用有效期
Callback string `json:"callback,omitempty"` // 回调地址
Key string `json:"key,omitempty"` // 文件标识符，通常为回调key
}
// UploadSession 上传会话
type UploadSession struct {
Key string
UID uint
PolicyID uint
VirtualPath string

11
routers/controllers/callback.go
View file

@ -76,3 +76,14 @@ func OneDriveCallback(c *gin.Context) {
c.JSON(200, ErrorResponse(err))
}
}
// COSCallback COS上传完成客户端回调
func COSCallback(c *gin.Context) {
var callbackBody callback.COSCallback
if err := c.ShouldBindQuery(&callbackBody); err == nil {
res := callbackBody.PreProcess(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}

6
routers/router.go
View file

@ -159,6 +159,12 @@ func InitMasterRouter() *gin.Engine {
controllers.OneDriveCallback,
)
}
// 腾讯云COS策略上传回调
callback.GET(
"cos/:key",
middleware.COSCallbackAuth(),
controllers.COSCallback,
)
}
// 需要登录保护的

46
service/callback/upload.go
View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/HFO4/cloudreve/pkg/filesystem"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/cos"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/onedrive"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
@ -52,6 +53,12 @@ type OneDriveCallback struct {
Meta *onedrive.FileInfo
}
// COSCallback COS 客户端回调正文
type COSCallback struct {
Bucket string `form:"bucket"`
Etag string `form:"etag"`
}
// GetBody 返回回调正文
func (service UpyunCallbackService) GetBody(session *serializer.UploadSession) serializer.UploadCallback {
res := serializer.UploadCallback{
@ -90,6 +97,16 @@ func (service OneDriveCallback) GetBody(session *serializer.UploadSession) seria
}
}
// GetBody 返回回调正文
func (service COSCallback) GetBody(session *serializer.UploadSession) serializer.UploadCallback {
return serializer.UploadCallback{
Name: session.Name,
SourceName: session.SavePath,
PicInfo: "",
Size: session.Size,
}
}
// ProcessCallback 处理上传结果回调
func ProcessCallback(service CallbackProcessService, c *gin.Context) serializer.Response {
// 创建文件系统
@ -168,9 +185,36 @@ func (service *OneDriveCallback) PreProcess(c *gin.Context) serializer.Response
// 验证与回调会话中是否一致
actualPath := strings.TrimPrefix(callbackSession.SavePath, "/")
if callbackSession.Size != info.Size || info.GetSourcePath() != actualPath {
// TODO 删除文件信息
fs.Handler.(onedrive.Driver).Client.Delete(context.Background(), []string{info.GetSourcePath()})
return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
}
service.Meta = info
return ProcessCallback(service, c)
}
// PreProcess 对COS客户端回调进行预处理
func (service *COSCallback) PreProcess(c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewFileSystemFromCallback(c)
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 获取回调会话
callbackSessionRaw, _ := c.Get("callbackSession")
callbackSession := callbackSessionRaw.(*serializer.UploadSession)
// 获取文件信息
info, err := fs.Handler.(cos.Driver).Meta(context.Background(), callbackSession.SavePath)
if err != nil {
return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
}
// 验证实际文件信息与回调会话中是否一致
if callbackSession.Size != info.Size || callbackSession.Key != info.CallbackKey {
return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
}
return ProcessCallback(service, c)
}