Feat: client-upload file in oss

2020-01-16 13:36:13 +08:00 · 2020-01-16 13:36:13 +08:00 · dd198becce
commit dd198becce
parent 0e62665d7f
13 changed files with 302 additions and 48 deletions
--- a/go.mod
+++ b/go.mod
@ -4,6 +4,7 @@ go 1.12

 require (
 	github.com/DATA-DOG/go-sqlmock v1.3.3
+	github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible
 	github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7
 	github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4
 	github.com/fatih/color v1.7.0
--- a/go.sum
+++ b/go.sum
@ -10,6 +10,8 @@ github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7I
 github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible h1:A3oZlWPD/Poa19FvNbw+Zu4yKAurDBTjlRDilYGBiS4=
+github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff h1:RmdPFa+slIr4SCBg4st/l/vZWVe9QJKMXGO60Bxbe04=
@ -213,6 +215,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
--- a/middleware/auth.go
+++ b/middleware/auth.go
@ -165,7 +165,6 @@ func RemoteCallbackAuth() gin.HandlerFunc {
 }

 // QiniuCallbackAuth 七牛回调签名验证
-// TODO 测试
 func QiniuCallbackAuth() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		// 验证key并查找用户
@ -194,3 +193,20 @@ func QiniuCallbackAuth() gin.HandlerFunc {
 		c.Next()
 	}
 }
+
+// OSSCallbackAuth 阿里云OSS回调签名验证
+func OSSCallbackAuth() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// 验证key并查找用户
+		resp, _ := uploadCallbackCheck(c)
+		if resp.Code != 0 {
+			c.JSON(401, serializer.QiniuCallbackFailed{Error: resp.Msg})
+			c.Abort()
+			return
+		}
+
+		// TODO 验证OSS给出的签名
+
+		c.Next()
+	}
+}
--- a/middleware/auth_test.go
+++ b/middleware/auth_test.go
@ -424,3 +424,53 @@ func TestQiniuCallbackAuth(t *testing.T) {
 		asserts.True(c.IsAborted())
 	}
 }
+
+func TestOSSCallbackAuth(t *testing.T) {
+	asserts := assert.New(t)
+	rec := httptest.NewRecorder()
+	AuthFunc := OSSCallbackAuth()
+
+	// Callback Key 相关验证失败
+	{
+		c, _ := gin.CreateTestContext(rec)
+		c.Params = []gin.Param{
+			{"key", "testOSSBackRemote"},
+		}
+		c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/testQiniuBackRemote", nil)
+		AuthFunc(c)
+		asserts.True(c.IsAborted())
+	}
+
+	// 成功
+	{
+		cache.Set(
+			"callback_testCallBackOSS",
+			serializer.UploadSession{
+				UID:         1,
+				PolicyID:    2,
+				VirtualPath: "/",
+			},
+			0,
+		)
+		cache.Deletes([]string{"1"}, "policy_")
+		mock.ExpectQuery("SELECT(.+)users(.+)").
+			WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
+		mock.ExpectQuery("SELECT(.+)groups(.+)").
+			WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[2]"))
+		mock.ExpectQuery("SELECT(.+)policies(.+)").
+			WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
+		c, _ := gin.CreateTestContext(rec)
+		c.Params = []gin.Param{
+			{"key", "testCallBackOSS"},
+		}
+		c.Request, _ = http.NewRequest("POST", "/api/v3/callback/qiniu/testCallBackOSS", nil)
+		mac := qbox.NewMac("123", "123")
+		token, err := mac.SignRequest(c.Request)
+		asserts.NoError(err)
+		c.Request.Header["Authorization"] = []string{"QBox " + token}
+		AuthFunc(c)
+		asserts.NoError(mock.ExpectationsWereMet())
+		asserts.False(c.IsAborted())
+	}
+
+}
--- a/models/policy.go
+++ b/models/policy.go
@ -111,7 +111,7 @@ func (policy *Policy) GeneratePath(uid uint, origin string) string {
 func (policy *Policy) GenerateFileName(uid uint, origin string) string {
 	// 未开启自动重命名时，直接返回原始文件名
 	if !policy.AutoRename {
-		return origin
+		return policy.getOriginNameRule(origin)
 	}

 	fileRule := policy.FileNameRule
@ -125,28 +125,31 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string {
 		"{date}":        time.Now().Format("20060102"),
 	}

+	replaceTable["{originname}"] = policy.getOriginNameRule(origin)
+
+	fileRule = util.Replace(replaceTable, fileRule)
+	return fileRule
+}
+
+func (policy Policy) getOriginNameRule(origin string) string {
 	// 部分存储策略可以使用{origin}代表原始文件名
 	if origin == "" {
 		// 如果上游未传回原始文件名，则使用占位符，让云存储端替换
 		switch policy.Type {
 		case "qiniu":
 			// 七牛会将$(fname)自动替换为原始文件名
-			replaceTable["{originname}"] = "$(fname)"
+			return "$(fname)"
 		case "local", "remote":
-			replaceTable["{originname}"] = origin
+			return origin
 		case "oss":
 			// OSS会将${filename}自动替换为原始文件名
-			replaceTable["{originname}"] = "${filename}"
+			return "${filename}"
 		case "upyun":
 			// Upyun会将{filename}{.suffix}自动替换为原始文件名
-			replaceTable["{originname}"] = "{filename}{.suffix}"
+			return "{filename}{.suffix}"
 		}
-	} else {
-		replaceTable["{originname}"] = origin
 	}
-
-	fileRule = util.Replace(replaceTable, fileRule)
-	return fileRule
+	return origin
 }

 // IsDirectlyPreview 返回此策略下文件是否可以直接预览（不需要重定向）
@ -172,6 +175,8 @@ func (policy *Policy) GetUploadURL() string {
 		controller, _ = url.Parse("/api/v3/file/upload")
 	case "remote":
 		controller, _ = url.Parse("/api/v3/slave/upload")
+	case "oss":
+		return policy.BaseURL
 	default:
 		controller, _ = url.Parse("")
 	}
--- a/models/policy_test.go
+++ b/models/policy_test.go
@ -91,47 +91,62 @@ func TestPolicy_GeneratePath(t *testing.T) {

 func TestPolicy_GenerateFileName(t *testing.T) {
 	asserts := assert.New(t)
-	testPolicy := Policy{
-		AutoRename: true,
+	// 重命名关闭
+	{
+		testPolicy := Policy{
+			AutoRename: false,
+		}
+		testPolicy.FileNameRule = "{randomkey16}"
+		asserts.Equal("123.txt", testPolicy.GenerateFileName(1, "123.txt"))
+
+		testPolicy.Type = "oss"
+		asserts.Equal("${filename}", testPolicy.GenerateFileName(1, ""))
 	}

-	testPolicy.FileNameRule = "{randomkey16}"
-	asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 16)
+	// 重命名开启
+	{
+		testPolicy := Policy{
+			AutoRename: true,
+		}

-	testPolicy.FileNameRule = "{randomkey8}"
-	asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8)
+		testPolicy.FileNameRule = "{randomkey16}"
+		asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 16)

-	testPolicy.FileNameRule = "{timestamp}"
-	asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.FormatInt(time.Now().Unix(), 10))
+		testPolicy.FileNameRule = "{randomkey8}"
+		asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8)

-	testPolicy.FileNameRule = "{uid}"
-	asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.Itoa(int(1)))
+		testPolicy.FileNameRule = "{timestamp}"
+		asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.FormatInt(time.Now().Unix(), 10))

-	testPolicy.FileNameRule = "{datetime}"
-	asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 14)
+		testPolicy.FileNameRule = "{uid}"
+		asserts.Equal(testPolicy.GenerateFileName(1, "123.txt"), strconv.Itoa(int(1)))

-	testPolicy.FileNameRule = "{date}"
-	asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8)
+		testPolicy.FileNameRule = "{datetime}"
+		asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 14)

-	testPolicy.FileNameRule = "123{date}ss{datetime}"
-	asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 27)
+		testPolicy.FileNameRule = "{date}"
+		asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 8)

-	// 支持{originname}的策略
-	testPolicy.Type = "local"
-	testPolicy.FileNameRule = "123{originname}"
-	asserts.Equal("123123.txt", testPolicy.GenerateFileName(1, "123.txt"))
+		testPolicy.FileNameRule = "123{date}ss{datetime}"
+		asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 27)

-	testPolicy.Type = "qiniu"
-	testPolicy.FileNameRule = "{uid}123{originname}"
-	asserts.Equal("1123123.txt", testPolicy.GenerateFileName(1, "123.txt"))
+		// 支持{originname}的策略
+		testPolicy.Type = "local"
+		testPolicy.FileNameRule = "123{originname}"
+		asserts.Equal("123123.txt", testPolicy.GenerateFileName(1, "123.txt"))

-	testPolicy.Type = "oss"
-	testPolicy.FileNameRule = "{uid}123{originname}"
-	asserts.Equal("1123${filename}", testPolicy.GenerateFileName(1, ""))
+		testPolicy.Type = "qiniu"
+		testPolicy.FileNameRule = "{uid}123{originname}"
+		asserts.Equal("1123123.txt", testPolicy.GenerateFileName(1, "123.txt"))

-	testPolicy.Type = "upyun"
-	testPolicy.FileNameRule = "{uid}123{originname}"
-	asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, ""))
+		testPolicy.Type = "oss"
+		testPolicy.FileNameRule = "{uid}123{originname}"
+		asserts.Equal("1123${filename}", testPolicy.GenerateFileName(1, ""))
+
+		testPolicy.Type = "upyun"
+		testPolicy.FileNameRule = "{uid}123{originname}"
+		asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, ""))
+	}

 }

--- a/pkg/filesystem/hooks_test.go
+++ b/pkg/filesystem/hooks_test.go
@ -414,6 +414,35 @@ func TestHookClearFileSize(t *testing.T) {

 }

+func TestHookUpdateSourceName(t *testing.T) {
+	asserts := assert.New(t)
+	fs := &FileSystem{User: &model.User{
+		Model: gorm.Model{ID: 1},
+	}}
+
+	// 成功
+	{
+		originFile := model.File{
+			Model:      gorm.Model{ID: 1},
+			SourceName: "new.txt",
+		}
+		ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
+		mock.ExpectBegin()
+		mock.ExpectExec("UPDATE(.+)").WithArgs("new.txt", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
+		mock.ExpectCommit()
+		err := HookUpdateSourceName(ctx, fs)
+		asserts.NoError(mock.ExpectationsWereMet())
+		asserts.NoError(err)
+	}
+
+	// 上下文错误
+	{
+		ctx := context.Background()
+		err := HookUpdateSourceName(ctx, fs)
+		asserts.Error(err)
+	}
+}
+
 func TestGenericAfterUpdate(t *testing.T) {
 	asserts := assert.New(t)
 	fs := &FileSystem{User: &model.User{
--- a/pkg/filesystem/oss/handller.go
+++ b/pkg/filesystem/oss/handller.go
@ -2,17 +2,64 @@ package oss

 import (
 	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
+	"fmt"
 	model "github.com/HFO4/cloudreve/models"
+	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
 	"github.com/HFO4/cloudreve/pkg/filesystem/response"
 	"github.com/HFO4/cloudreve/pkg/serializer"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"io"
 	"net/url"
+	"path"
+	"time"
 )

+// UploadPolicy 阿里云OSS上传策略
+type UploadPolicy struct {
+	Expiration string        `json:"expiration"`
+	Conditions []interface{} `json:"conditions"`
+}
+
+// CallbackPolicy 回调策略
+type CallbackPolicy struct {
+	CallbackURL      string `json:"callbackUrl"`
+	CallbackBody     string `json:"callbackBody"`
+	CallbackBodyType string `json:"callbackBodyType"`
+}
+
 // Handler 阿里云OSS策略适配器
 type Handler struct {
 	Policy *model.Policy
+	client *oss.Client
+	bucket *oss.Bucket
+}
+
+// InitOSSClient 初始化OSS鉴权客户端
+func (handler *Handler) InitOSSClient() error {
+	if handler.Policy == nil {
+		return errors.New("存储策略为空")
+	}
+
+	// 初始化客户端
+	client, err := oss.New(handler.Policy.Server, handler.Policy.AccessKey, handler.Policy.SecretKey)
+	if err != nil {
+		return err
+	}
+	handler.client = client
+
+	// 初始化存储桶
+	bucket, err := client.Bucket(handler.Policy.BucketName)
+	if err != nil {
+		return err
+	}
+	handler.bucket = bucket
+
+	return nil
 }

 // Get 获取文件
@ -50,5 +97,74 @@ func (handler Handler) Source(

 // Token 获取上传策略和认证Token
 func (handler Handler) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
-	return serializer.UploadCredential{}, errors.New("未实现")
+	// 读取上下文中生成的存储路径
+	savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
+	if !ok {
+		return serializer.UploadCredential{}, errors.New("无法获取存储路径")
+	}
+
+	// 生成回调地址
+	siteURL := model.GetSiteURL()
+	apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + key)
+	apiURL := siteURL.ResolveReference(apiBaseURI)
+
+	// 回调策略
+	callbackPolicy := CallbackPolicy{
+		CallbackURL:      apiURL.String(),
+		CallbackBody:     `{"name":${x:fname},"source_name":${object},"size":${size},"pic_info":"${imageInfo.width},${imageInfo.height}"}`,
+		CallbackBodyType: "application/json",
+	}
+
+	// 上传策略
+	postPolicy := UploadPolicy{
+		Expiration: time.Now().UTC().Add(time.Duration(TTL) * time.Second).Format(time.RFC3339),
+		Conditions: []interface{}{
+			map[string]string{"bucket": handler.Policy.BucketName},
+			[]string{"starts-with", "$key", path.Dir(savePath)},
+			[]interface{}{"content-length-range", 0, handler.Policy.MaxSize},
+		},
+	}
+
+	return handler.getUploadCredential(ctx, postPolicy, callbackPolicy, TTL)
+}
+
+func (handler Handler) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64) (serializer.UploadCredential, error) {
+	// 读取上下文中生成的存储路径
+	savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
+	if !ok {
+		return serializer.UploadCredential{}, errors.New("无法获取存储路径")
+	}
+
+	// 处理回调策略
+	callbackPolicyEncoded := ""
+	if callback.CallbackURL != "" {
+		callbackPolicyJSON, err := json.Marshal(callback)
+		if err != nil {
+			return serializer.UploadCredential{}, err
+		}
+		callbackPolicyEncoded = base64.StdEncoding.EncodeToString(callbackPolicyJSON)
+		policy.Conditions = append(policy.Conditions, map[string]string{"callback": callbackPolicyEncoded})
+	}
+
+	// 编码上传策略
+	policyJSON, err := json.Marshal(policy)
+	if err != nil {
+		return serializer.UploadCredential{}, err
+	}
+	policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
+
+	// 签名上传策略
+	hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
+	_, err = io.WriteString(hmacSign, policyEncoded)
+	if err != nil {
+		return serializer.UploadCredential{}, err
+	}
+	signature := base64.StdEncoding.EncodeToString(hmacSign.Sum(nil))
+
+	return serializer.UploadCredential{
+		Policy:    fmt.Sprintf("%s:%s", callbackPolicyEncoded, policyEncoded),
+		Path:      savePath,
+		AccessKey: handler.Policy.AccessKey,
+		Token:     signature,
+	}, nil
 }
--- a/pkg/serializer/upload.go
+++ b/pkg/serializer/upload.go
@ -18,8 +18,10 @@ type UploadPolicy struct {

 // UploadCredential 返回给客户端的上传凭证
 type UploadCredential struct {
-	Token  string `json:"token"`
-	Policy string `json:"policy"`
+	Token     string `json:"token"`
+	Policy    string `json:"policy"`
+	Path      string `json:"path"`
+	AccessKey string `json:"ak"`
 }

 // UploadSession 上传会话
--- a/routers/controllers/callback.go
+++ b/routers/controllers/callback.go
@ -19,7 +19,7 @@ func RemoteCallback(c *gin.Context) {

 // QiniuCallback 七牛上传回调
 func QiniuCallback(c *gin.Context) {
-	var callbackBody callback.QiniuUploadCallbackService
+	var callbackBody callback.UploadCallbackService
 	if err := c.ShouldBindJSON(&callbackBody); err == nil {
 		res := callback.ProcessCallback(callbackBody, c)
 		if res.Code != 0 {
@ -31,3 +31,14 @@ func QiniuCallback(c *gin.Context) {
 		c.JSON(401, ErrorResponse(err))
 	}
 }
+
+// OSSCallback 阿里云OSS上传回调
+func OSSCallback(c *gin.Context) {
+	var callbackBody callback.UploadCallbackService
+	if err := c.ShouldBindJSON(&callbackBody); err == nil {
+		res := callback.ProcessCallback(callbackBody, c)
+		c.JSON(200, res)
+	} else {
+		c.JSON(200, ErrorResponse(err))
+	}
+}
--- a/routers/router.go
+++ b/routers/router.go
@ -138,6 +138,12 @@ func InitMasterRouter() *gin.Engine {
 				middleware.QiniuCallbackAuth(),
 				controllers.QiniuCallback,
 			)
+			// 阿里云OSS策略上传回调
+			callback.POST(
+				"oss/:key",
+				middleware.OSSCallbackAuth(),
+				controllers.OSSCallback,
+			)
 		}

 		// 需要登录保护的
--- a/service/callback/upload.go
+++ b/service/callback/upload.go
@ -25,8 +25,8 @@ func (service RemoteUploadCallbackService) GetBody() serializer.UploadCallback {
 	return service.Data
 }

-// QiniuUploadCallbackService 七牛存储上传回调请求服务
-type QiniuUploadCallbackService struct {
+// UploadCallbackService 云存储上传回调请求服务
+type UploadCallbackService struct {
 	Name       string `json:"name"`
 	SourceName string `json:"source_name"`
 	PicInfo    string `json:"pic_info"`
@ -34,7 +34,7 @@ type QiniuUploadCallbackService struct {
 }

 // GetBody 返回回调正文
-func (service QiniuUploadCallbackService) GetBody() serializer.UploadCallback {
+func (service UploadCallbackService) GetBody() serializer.UploadCallback {
 	return serializer.UploadCallback{
 		Name:       service.Name,
 		SourceName: service.SourceName,
--- a/service/explorer/upload.go
+++ b/service/explorer/upload.go
@ -11,7 +11,7 @@ import (
 // UploadCredentialService 获取上传凭证服务
 type UploadCredentialService struct {
 	Path string `form:"path" binding:"required"`
-	Size uint64 `form:"size" binding:"required,min=0"`
+	Size uint64 `form:"size" binding:"min=0"`
 }

 // Get 获取新的上传凭证