Infrastructure/Cloudreve
0
Fork 0
Code Issues Pull requests Activity

Feat: put / get file in qiniu policy

Browse source
This commit is contained in:
HFO4 2020-01-15 16:03:23 +08:00
parent 7fbbcefc57
commit 90f82100cf
8 changed files with 158 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
models/policy.go
View file

@ -126,18 +126,23 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string {
}
// 部分存储策略可以使用{origin}代表原始文件名
switch policy.Type {
case "qiniu":
// 七牛会将$(fname)自动替换为原始文件名
replaceTable["{originname}"] = "$(fname)"
case "local", "remote":
if origin == "" {
// 如果上游未传回原始文件名，则使用占位符，让云存储端替换
switch policy.Type {
case "qiniu":
// 七牛会将$(fname)自动替换为原始文件名
replaceTable["{originname}"] = "$(fname)"
case "local", "remote":
replaceTable["{originname}"] = origin
case "oss":
// OSS会将${filename}自动替换为原始文件名
replaceTable["{originname}"] = "${filename}"
case "upyun":
// Upyun会将{filename}{.suffix}自动替换为原始文件名
replaceTable["{originname}"] = "{filename}{.suffix}"
}
} else {
replaceTable["{originname}"] = origin
case "oss":
// OSS会将${filename}自动替换为原始文件名
replaceTable["{originname}"] = "${filename}"
case "upyun":
// Upyun会将{filename}{.suffix}自动替换为原始文件名
replaceTable["{originname}"] = "{filename}{.suffix}"
}
fileRule = util.Replace(replaceTable, fileRule)

3
models/policy_test.go
View file

@ -123,7 +123,7 @@ func TestPolicy_GenerateFileName(t *testing.T) {
testPolicy.Type = "qiniu"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123$(fname)", testPolicy.GenerateFileName(1, "123.txt"))
asserts.Equal("1123123.txt", testPolicy.GenerateFileName(1, "123.txt"))
testPolicy.Type = "oss"
testPolicy.FileNameRule = "{uid}123{originname}"
@ -132,6 +132,7 @@ func TestPolicy_GenerateFileName(t *testing.T) {
testPolicy.Type = "upyun"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, ""))
}
func TestPolicy_IsDirectlyPreview(t *testing.T) {

5
pkg/filesystem/archive.go
View file

@ -112,7 +112,10 @@ func (fs *FileSystem) doCompress(ctx context.Context, file *model.File, folder *
}
// 获取文件内容
fileToZip, err := fs.Handler.Get(ctx, file.SourceName)
fileToZip, err := fs.Handler.Get(
context.WithValue(ctx, fsctx.FileModelCtx, *file),
file.SourceName,
)
if err != nil {
util.Log().Debug("Open%s，%s", file.Name, err)
return

42
pkg/filesystem/qiniu/handler_test.go
View file

@ -1,42 +0,0 @@
package qiniu
import (
"context"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/stretchr/testify/assert"
"testing"
)
func TestHandler_Token(t *testing.T) {
asserts := assert.New(t)
handler := Handler{
Policy: &model.Policy{
MaxSize: 10,
OptionsSerialized: model.PolicyOption{
MimeType: "ss",
},
AccessKey: "ak",
SecretKey: "sk",
Server: "http://test.com",
},
}
ctx := context.Background()
// 成功
{
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/123")
_, err := handler.Token(ctx, 10, "123")
asserts.NoError(err)
}
// 上下文无存储路径
{
ctx = context.Background()
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
_, err := handler.Token(ctx, 10, "123")
asserts.Error(err)
}
}

135
pkg/filesystem/qiniu/handller.go
View file

@ -7,10 +7,12 @@ import (
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/request"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/qiniu/api.v7/v7/storage"
"io"
"net/http"
"net/url"
"time"
)
@ -22,43 +24,118 @@ type Handler struct {
// Get 获取文件
func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, error) {
return nil, errors.New("未实现")
// 给文件名加上随机参数以强制拉取
path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano())
// 获取文件源地址
downloadURL, err := handler.Source(
ctx,
path,
url.URL{},
int64(model.GetIntSetting("preview_timeout", 60)),
false,
0,
)
if err != nil {
return nil, err
}
// 获取文件数据流
client := request.HTTPClient{}
resp, err := client.Request(
"GET",
downloadURL,
nil,
request.WithContext(ctx),
request.WithHeader(
http.Header{"Cache-Control": {"no-cache", "no-store", "must-revalidate"}},
),
).CheckHTTPResponse(200).GetRSCloser()
if err != nil {
return nil, err
}
resp.SetFirstFakeChunk()
// 尝试自主获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}
// Put 将文件流保存到指定目录
func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
return errors.New("未实现")
//// 凭证生成
//putPolicy := storage.PutPolicy{
// Scope: "cloudrevetest",
//}
//mac := auth.New("YNzTBBpDUq4EEiFV0-vyJCZCJ0LvUEI0_WvxtEXE", "Clm9d9M2CH7pZ8vm049ZlGZStQxrRQVRTjU_T5_0")
//upToken := putPolicy.UploadToken(mac)
//
//cfg := storage.Config{}
//// 空间对应的机房
//cfg.Zone = &storage.ZoneHuadong
//formUploader := storage.NewFormUploader(&cfg)
//ret := storage.PutRet{}
//putExtra := storage.PutExtra{
// Params: map[string]string{},
//}
//
//defer file.Close()
//
//err := formUploader.Put(ctx, &ret, upToken, dst, file, int64(size), &putExtra)
//if err != nil {
// fmt.Println(err)
// return err
//}
//fmt.Println(ret.Key, ret.Hash)
//return nil
defer file.Close()
// 凭证有效期
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
// 生成上传策略
putPolicy := storage.PutPolicy{
// 指定为覆盖策略
Scope: fmt.Sprintf("%s:%s", handler.Policy.BucketName, dst),
SaveKey: dst,
ForceSaveKey: true,
FsizeLimit: int64(size),
}
// 是否开启了MIMEType限制
if handler.Policy.OptionsSerialized.MimeType != "" {
putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType
}
// 生成上传凭证
token, err := handler.getUploadCredential(ctx, putPolicy, int64(credentialTTL))
if err != nil {
return err
}
// 创建上传表单
cfg := storage.Config{}
formUploader := storage.NewFormUploader(&cfg)
ret := storage.PutRet{}
putExtra := storage.PutExtra{
Params: map[string]string{},
}
// 开始上传
err = formUploader.Put(ctx, &ret, token.Token, dst, file, int64(size), &putExtra)
if err != nil {
return err
}
return nil
}
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
// 返回未删除的文件
func (handler Handler) Delete(ctx context.Context, files []string) ([]string, error) {
return []string{}, errors.New("未实现")
// TODO 大于一千个文件需要分批发送
deleteOps := make([]string, 0, len(files))
for _, key := range files {
deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key))
}
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
cfg := storage.Config{
UseHTTPS: true,
}
bucketManager := storage.NewBucketManager(mac, &cfg)
rets, err := bucketManager.Batch(deleteOps)
// 处理删除结果
if err != nil {
failed := make([]string, 0, len(rets))
for k, ret := range rets {
if ret.Code != 200 {
failed = append(failed, files[k])
}
}
return failed, errors.New("删除失败")
}
return []string{}, nil
}
// Thumb 获取文件缩略图

6
pkg/filesystem/remote/handler.go
View file

@ -76,6 +76,12 @@ func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser,
}
resp.SetFirstFakeChunk()
// 尝试获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}

20
pkg/request/request.go
View file

@ -191,7 +191,6 @@ func (resp *Response) DecodeResponse() (*serializer.Response, error) {
// NopRSCloser 实现不完整seeker
type NopRSCloser struct {
body io.ReadCloser
size int64
status *rscStatus
}
@ -199,6 +198,8 @@ type rscStatus struct {
// http.ServeContent 会读取一小块以决定内容类型，
// 但是响应body无法实现seek，所以此项为真时第一个read会返回假数据
IgnoreFirst bool
Size int64
}
// GetRSCloser 返回带有空seeker的RSCloser，供http.ServeContent使用
@ -208,9 +209,10 @@ func (resp *Response) GetRSCloser() (*NopRSCloser, error) {
}
return &NopRSCloser{
body: resp.Response.Body,
size: resp.Response.ContentLength,
status: &rscStatus{},
body: resp.Response.Body,
status: &rscStatus{
Size: resp.Response.ContentLength,
},
}, resp.Err
}
@ -220,11 +222,13 @@ func (instance NopRSCloser) SetFirstFakeChunk() {
instance.status.IgnoreFirst = true
}
// SetContentLength 设置数据流大小
func (instance NopRSCloser) SetContentLength(size int64) {
instance.status.Size = size
}
// Read 实现 NopRSCloser reader
func (instance NopRSCloser) Read(p []byte) (n int, err error) {
if instance.status.IgnoreFirst {
return 0, io.EOF
}
return instance.body.Read(p)
}
@ -244,7 +248,7 @@ func (instance NopRSCloser) Seek(offset int64, whence int) (int64, error) {
case io.SeekStart:
return 0, nil
case io.SeekEnd:
return instance.size, nil
return instance.status.Size, nil
}
}
return 0, errors.New("未实现")

12
pkg/request/request_test.go
View file

@ -211,3 +211,15 @@ func TestResponse_DecodeResponse(t *testing.T) {
asserts.Equal(0, response.Code)
}
}
func TestNopRSCloser_SetFirstFakeChunk(t *testing.T) {
asserts := assert.New(t)
rsc := NopRSCloser{
status: &rscStatus{},
}
rsc.SetFirstFakeChunk()
asserts.True(rsc.status.IgnoreFirst)
rsc.SetContentLength(20)
asserts.EqualValues(20, rsc.status.Size)
}