diff --git a/go.mod b/go.mod index b2154cc..4d3a11d 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/DATA-DOG/go-sqlmock v1.3.3 github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible + github.com/aws/aws-sdk-go v1.31.9 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4 github.com/fatih/color v1.7.0 @@ -15,7 +16,7 @@ require ( github.com/gin-gonic/gin v1.5.0 github.com/go-ini/ini v1.50.0 github.com/go-mail/mail v2.3.1+incompatible - github.com/gofrs/uuid v3.2.0+incompatible + github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/gomodule/redigo v2.0.0+incompatible github.com/google/go-querystring v1.0.0 github.com/gorilla/websocket v1.4.1 @@ -25,17 +26,17 @@ require ( github.com/mattn/go-colorable v0.1.4 // indirect github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 - github.com/pkg/errors v0.8.0 + github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.2.0 - github.com/qingwg/payjs v0.0.0-20190928033402-c53dbe16b371 + github.com/qingwg/payjs v0.0.0-20190928033402-c53dbe16b371 // indirect github.com/qiniu/api.v7/v7 v7.4.0 github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1 github.com/rakyll/statik v0.1.7 github.com/robfig/cron/v3 v3.0.1 - github.com/smartwalle/alipay/v3 v3.0.13 + github.com/smartwalle/alipay/v3 v3.0.13 // indirect github.com/smartystreets/goconvey v1.6.4 // indirect github.com/speps/go-hashids v2.0.0+incompatible - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.5.1 github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac github.com/upyun/go-sdk v2.1.0+incompatible diff --git a/go.sum b/go.sum index 8393d9d..2f14014 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible h1:A3oZlWPD/Poa19FvNbw+Zu4yKAurDBTjlRDilYGBiS4= github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/aws/aws-sdk-go v1.31.9 h1:n+b34ydVfgC30j0Qm69yaapmjejQPW2BoDBX7Uy/tLI= +github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -75,6 +77,8 @@ github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rm github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -128,6 +132,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -186,6 +192,8 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/otp v1.2.0 h1:/A3+Jn+cagqayeR3iHs/L62m5ue7710D35zl1zJ1kok= @@ -231,6 +239,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible h1:dqpmYaez7VBT7PCRBcBxkzlDOiTk7Td8ATiia1b1GuE= github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ= @@ -268,6 +278,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/models/policy.go b/models/policy.go index a4d4181..0170989 100644 --- a/models/policy.go +++ b/models/policy.go @@ -58,6 +58,7 @@ var thumbSuffix = map[string][]string{ "upyun": {".svg", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"}, "remote": {}, "onedrive": {"*"}, + "bos": {".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"}, } func init() { @@ -203,7 +204,7 @@ func (policy *Policy) IsThumbExist(name string) bool { // IsTransitUpload 返回此策略上传给定size文件时是否需要服务端中转 func (policy *Policy) IsTransitUpload(size uint64) bool { - if policy.Type == "local" { + if policy.Type == "local" || policy.Type == "bos" { return true } if policy.Type == "onedrive" && size < 4*1024*1024 { @@ -236,7 +237,7 @@ func (policy *Policy) GetUploadURL() string { var controller *url.URL switch policy.Type { - case "local", "onedrive": + case "local", "onedrive", "bos": return "/api/v3/file/upload" case "remote": controller, _ = url.Parse("/api/v3/slave/upload") diff --git a/pkg/filesystem/driver/bos/handler.go b/pkg/filesystem/driver/bos/handler.go new file mode 100644 index 0000000..3cc4ec0 --- /dev/null +++ b/pkg/filesystem/driver/bos/handler.go @@ -0,0 +1,256 @@ +package bos + +import ( + "context" + "errors" + "fmt" + model "github.com/HFO4/cloudreve/models" + "github.com/HFO4/cloudreve/pkg/auth" + "github.com/HFO4/cloudreve/pkg/cache" + "github.com/HFO4/cloudreve/pkg/filesystem/fsctx" + "github.com/HFO4/cloudreve/pkg/filesystem/response" + "github.com/HFO4/cloudreve/pkg/serializer" + "github.com/HFO4/cloudreve/pkg/util" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/endpoints" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "io" + "net/url" + "os" + "path/filepath" +) + +var UserPathPrefix = "user" + +// Driver 策略适配器 +type Driver struct { + Policy *model.Policy + ss *session.Session +} + +// InitSess 初始化会话 +func (handler *Driver) connect() { + if nil == handler.Policy { + panic(errors.New("存储策略为空")) + } + + handler.ss = session.Must(session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(handler.Policy.AccessKey, handler.Policy.SecretKey, ""), + Endpoint: aws.String(handler.Policy.Server), + Region: aws.String(endpoints.CnNorth1RegionID), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), // 这个必须设置为TRUE，坑了老半天 + })) +} + +// 刷新 S3 Gateway 服务的订单信息 +func (handler *Driver) init() error { + if nil == handler.ss { + handler.connect() + } + + svc := s3.New(handler.ss) + _, err := svc.ListBuckets(nil) + + return err +} + +func prefix(path string) string { + return UserPathPrefix + path +} + +// List 列出BOS上存储的文件 +func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) { + var res []response.Object + // 刷新Buckets + if err := handler.init(); nil != err { + return nil, err + } + + svc := s3.New(handler.ss) + resp, err := svc.ListObjects(&s3.ListObjectsInput{ + Bucket: aws.String(handler.Policy.BucketName), + Prefix: aws.String(prefix(path)), + }) + if nil != err { + return nil, err + } + + for _, item := range resp.Contents { + res = append(res, response.Object{ + Name: filepath.Base(aws.StringValue(item.Key)), + Size: uint64(aws.Int64Value(item.Size)), + LastModify: aws.TimeValue(item.LastModified), + RelativePath: filepath.ToSlash(aws.StringValue(item.Key)), + }) + } + + return res, nil +} + +// Get 获取文件 +func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { + // 刷新Buckets + if err := handler.init(); nil != err { + return nil, err + } + + file, err := os.Create(filepath.Base(path)) + if nil != err { + return nil, err + } + + downloader := s3manager.NewDownloader(handler.ss) + _, err = downloader.Download(file, &s3.GetObjectInput{ + Bucket: aws.String(handler.Policy.BucketName), + Key: aws.String(prefix(path)), + }) + + if nil != err { + return nil, err + } + + return file, nil +} + +// Put 将文件流保存到指定目录 +func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error { + // 刷新Buckets + if err := handler.init(); nil != err { + return err + } + + uploader := s3manager.NewUploader(handler.ss) + _, err := uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(handler.Policy.BucketName), + Key: aws.String(prefix(dst)), + Body: file, + }) + + return err +} + +// Delete 删除一个或多个文件， +// 返回未删除的文件 +func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) { + return files, errors.New("Lambda Chain 暂不支持删除操作") +} + +// Thumb 获取文件缩略图 +func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) { + //if nil == handler.ss { + // handler.connect() + //} + // + //svc := s3.New(handler.ss) + //result, err := svc.ListBuckets(nil) + //if nil != err { + // return nil, err + //} + // + //for _, b := range result.Buckets { + // log.Println(aws.StringValue(b.Name)) + // + // resp, err := svc.ListObjects(&s3.ListObjectsInput{ + // Bucket: b.Name, + // }) + // + // if nil != err { + // log.Println("Error:", err) + // continue + // } + // + // for _, item := range resp.Contents { + // log.Println("Key:", aws.StringValue(item.Key)) + // log.Println("Size:", aws.Int64Value(item.Size)) + // log.Println("Class:", aws.StringValue(item.StorageClass)) + // log.Println("LastTime:", aws.TimeValue(item.LastModified)) + // } + //} + return &response.ContentResponse{ + Redirect: true, + URL: "", + }, nil +} + +// Source 获取外链URL +func (handler Driver) Source(ctx context.Context, path string, baseURL url.URL, ttl int64, isDownload bool, speed int) (string, error) { + file, ok := ctx.Value(fsctx.FileModelCtx).(model.File) + if !ok { + return "", errors.New("无法获取文件记录上下文") + } + + var ( + signedURI *url.URL + err error + ) + if isDownload { + // 创建下载会话，将文件信息写入缓存 + downloadSessionID := util.RandStringRunes(16) + err = cache.Set("download_"+downloadSessionID, file, int(ttl)) + if err != nil { + return "", serializer.NewError(serializer.CodeCacheOperation, "无法创建下载会话", err) + } + + // 签名生成文件记录 + signedURI, err = auth.SignURI( + auth.General, + fmt.Sprintf("/api/v3/file/download/%s", downloadSessionID), + ttl, + ) + } + + if nil != err { + return "", serializer.NewError(serializer.CodeEncryptError, "无法对URL进行签名", err) + + } + + finalURL := baseURL.ResolveReference(signedURI).String() + + return finalURL, nil +} + +// Token 获取上传策略和认证Token +func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) { + // 生成回调地址 + siteURL := model.GetSiteURL() + apiBaseURI, _ := url.Parse("/api/v3/callback/bos/" + key) + apiURL := siteURL.ResolveReference(apiBaseURI).String() + + return serializer.UploadCredential{ + Callback: apiURL, + Key: key, + }, nil +} + +// Meta 获取文件信息用于回调验证 +// Cloudreve 本身不允许同名文件存在，所以通过Cloudreve上传到 +// Lambda BOS存储的文件原则上也不会有重名，所以用文件名称和大 +// 小做验证信息，比较粗劣，后续优化 +//func (handler Driver) Meta(path string) (uint64, error) { +// // 刷新Buckets +// if err := handler.init(); nil != err { +// return 0, err +// } +// +// dir, _ := filepath.Split(path) +// svc := s3.New(handler.ss) +// resp, err := svc.ListObjects(&s3.ListObjectsInput{ +// Bucket: aws.String(handler.Policy.BucketName), +// Prefix: aws.String(prefix(dir)), +// }) +// if nil != err { +// return 0, err +// } +// +// for _, item := range resp.Contents { +// if path == aws.StringValue(item.Key) { +// return uint64(aws.Int64Value(item.Size)), nil +// } +// } +// +// return 0, errors.New("文件验证异常") +//} diff --git a/pkg/filesystem/driver/bos/handler_test.go b/pkg/filesystem/driver/bos/handler_test.go new file mode 100644 index 0000000..e8ef6a1 --- /dev/null +++ b/pkg/filesystem/driver/bos/handler_test.go @@ -0,0 +1,90 @@ +package bos + +import ( + "context" + model "github.com/HFO4/cloudreve/models" + "github.com/stretchr/testify/assert" + "io/ioutil" + "log" + "strings" + "testing" +) + +func TestDriver_Get(t *testing.T) { + asserts := assert.New(t) + handler := Driver{ + Policy: &model.Policy{ + AccessKey: "accesskey", + SecretKey: "secretkey", + BucketName: "file", + Server: "http://39.105.48.70:9002", + }, + } + path := "TestFile.txt_44c873c0fe0a4733a28e701c4e24cf6d" + resp, err := handler.Get(context.Background(), path) + asserts.Error(err) + asserts.Nil(resp) +} + +func TestDriver_Put(t *testing.T) { + asserts := assert.New(t) + handler := Driver{ + Policy: &model.Policy{ + AccessKey: "accesskey", + SecretKey: "secretkey", + BucketName: "file", + Server: "http://39.105.48.70:9002", + }, + } + + dst := "TestFile.txt" + err := handler.Put(context.Background(), ioutil.NopCloser(strings.NewReader("666")), dst, 3) + asserts.Error(err) +} + +func TestDriver_Thumb(t *testing.T) { + asserts := assert.New(t) + handler := Driver{ + Policy: &model.Policy{ + AccessKey: "accesskey", + SecretKey: "secretkey", + Server: "http://39.105.48.70:9002", + }, + } + + resp, err := handler.Thumb(context.Background(), "./999.txt") + asserts.Error(err) + asserts.Nil(resp) +} + +func TestDriver_Delete(t *testing.T) { + asserts := assert.New(t) + handler := Driver{ + Policy: &model.Policy{ + AccessKey: "accesskey", + SecretKey: "secretkey", + BucketName: "file", + Server: "http://39.105.48.70:9002", + }, + } + + resp, err := handler.Delete(context.Background(), []string{"TestFile.txt_3e46f40e0e774376979add820f142653"}) + asserts.Error(err) + log.Println(resp) +} + +func TestDriver_List(t *testing.T) { + asserts := assert.New(t) + handler := Driver{ + Policy: &model.Policy{ + AccessKey: "accesskey", + SecretKey: "secretkey", + BucketName: "439C4A6DC3F38B825D03F0357729A1E22B39FFCE", + Server: "http://123.56.171.188:9002", + }, + } + + resp, err := handler.List(context.Background(), "1", false) + asserts.Error(err) + log.Println(resp) +} diff --git a/pkg/filesystem/filesystem.go b/pkg/filesystem/filesystem.go index 3e1fac5..c40d579 100644 --- a/pkg/filesystem/filesystem.go +++ b/pkg/filesystem/filesystem.go @@ -6,6 +6,7 @@ import ( "github.com/HFO4/cloudreve/models" "github.com/HFO4/cloudreve/pkg/auth" "github.com/HFO4/cloudreve/pkg/conf" + "github.com/HFO4/cloudreve/pkg/filesystem/driver/bos" "github.com/HFO4/cloudreve/pkg/filesystem/driver/cos" "github.com/HFO4/cloudreve/pkg/filesystem/driver/local" "github.com/HFO4/cloudreve/pkg/filesystem/driver/onedrive" @@ -219,6 +220,10 @@ func (fs *FileSystem) DispatchHandler() error { HTTPClient: request.HTTPClient{}, } return nil + case "bos": + fs.Handler = bos.Driver{ + Policy: currentPolicy, + } default: return ErrUnknownPolicyType }