Infrastructure/Cloudreve
0
Fork 0
Code Issues Pull requests Activity

Feat: remote delete file / Fix: broken Filesystem recycle in upload request

Browse source
This commit is contained in:
HFO4 2020-01-01 20:15:05 +08:00
parent e9f02940ee
commit ee08821361
14 changed files with 152 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
pkg/filesystem/filesystem.go
View file

@ -120,6 +120,9 @@ func NewAnonymousFileSystem() (*FileSystem, error) {
return nil, err
}
fs.User.Group = anonymousGroup
} else {
// 从机模式下，分配本地策略处理器
fs.Handler = local.Handler{}
}
return fs, nil

8
pkg/filesystem/local/handler.go
View file

@ -69,7 +69,10 @@ func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string,
util.Log().Warning("无法创建文件，%s", err)
return err
}
defer out.Close()
defer func() {
err := out.Close()
fmt.Print(err)
}()
// 写入文件内容
_, err = io.Copy(out, file)
@ -138,7 +141,7 @@ func (handler Handler) Source(
downloadSessionID := util.RandStringRunes(16)
err = cache.Set("download_"+downloadSessionID, file, int(ttl))
if err != nil {
return "", serializer.NewError(serializer.CodeCacheOperation, "无法创建下載会话", err)
return "", serializer.NewError(serializer.CodeCacheOperation, "无法创建下载会话", err)
}
// 签名生成文件记录
@ -165,7 +168,6 @@ func (handler Handler) Source(
}
// Token 获取上传策略和认证Token，本地策略直接返回空值
// TODO 测试
func (handler Handler) Token(ctx context.Context, ttl int64, key string) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, nil
}

14
pkg/filesystem/manage.go
View file

@ -131,6 +131,7 @@ func (fs *FileSystem) Move(ctx context.Context, dirs, files []uint, src, dst str
func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint) error {
// 已删除的总容量,map用于去重
var deletedStorage = make(map[uint]uint64)
var totalStorage = make(map[uint]uint64)
// 已删除的文件ID
var deletedFileIDs = make([]uint, 0, len(fs.FileTarget))
// 删除失败的文件的父目录ID
@ -166,15 +167,18 @@ func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint) error {
// 按照存储策略分组删除对象
failed := fs.deleteGroupedFile(ctx, policyGroup)
// 整理删除结果
for i := 0; i < len(fs.FileTarget); i++ {
if util.ContainsString(failed[fs.FileTarget[i].PolicyID], fs.FileTarget[i].SourceName) {
// TODO 删除失败时不删除文件记录及父目录
} else {
if !util.ContainsString(failed[fs.FileTarget[i].PolicyID], fs.FileTarget[i].SourceName) {
// 已成功删除的文件
deletedFileIDs = append(deletedFileIDs, fs.FileTarget[i].ID)
deletedStorage[fs.FileTarget[i].ID] = fs.FileTarget[i].Size
}
deletedStorage[fs.FileTarget[i].ID] = fs.FileTarget[i].Size
// 全部文件
totalStorage[fs.FileTarget[i].ID] = fs.FileTarget[i].Size
allFileIDs = append(allFileIDs, fs.FileTarget[i].ID)
}
// TODO 用户自主选择是否强制删除
// 删除文件记录
err = model.DeleteFileByIDs(allFileIDs)
@ -184,7 +188,7 @@ func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint) error {
// 归还容量
var total uint64
for _, value := range deletedStorage {
for _, value := range totalStorage {
total += value
}
fs.User.DeductionStorage(total)

58
pkg/filesystem/remote/handler.go
View file

@ -4,24 +4,44 @@ package remote
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/auth"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/request"
"github.com/HFO4/cloudreve/pkg/serializer"
"io"
"net/http"
"net/url"
"strings"
"time"
)
// Handler 远程存储策略适配器
type Handler struct {
client request.HTTPClient
Policy *model.Policy
}
// getAPI 获取接口请求地址
func (handler Handler) getAPI(scope string) string {
serverURL, err := url.Parse(handler.Policy.Server)
if err != nil {
return ""
}
var controller *url.URL
switch scope {
case "delete":
controller, _ = url.Parse("/api/v3/slave/delete")
}
return serverURL.ResolveReference(controller).String()
}
// Get 获取文件内容
func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, error) {
@ -35,7 +55,45 @@ func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string,
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
// TODO 测试
func (handler Handler) Delete(ctx context.Context, files []string) ([]string, error) {
// 封装接口请求正文
reqBody := serializer.RemoteDeleteRequest{
Files: files,
}
reqBodyEncoded, err := json.Marshal(reqBody)
if err != nil {
return files, err
}
// 发送删除请求
bodyReader := strings.NewReader(string(reqBodyEncoded))
authInstance := auth.HMACAuth{SecretKey: []byte(handler.Policy.SecretKey)}
resp, err := handler.client.Request(
"POST",
handler.getAPI("delete"),
bodyReader,
request.WithCredential(authInstance, 60),
).GetResponse(200)
if err != nil {
return files, err
}
// 处理删除结果
var reqResp serializer.Response
err = json.Unmarshal([]byte(resp), &reqResp)
if err != nil {
return files, err
}
if reqResp.Code != 0 {
var failedResp serializer.RemoteDeleteRequest
err = json.Unmarshal([]byte(reqResp.Data.(string)), &failedResp)
if err == nil {
return failedResp.Files, errors.New(reqResp.Error)
}
return files, errors.New("未知的返回结果格式")
}
return []string{}, nil
}

1
pkg/filesystem/upload.go
View file

@ -116,6 +116,7 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file FileHe
} else {
reqContext = ctx.Value(fsctx.HTTPCtx).(context.Context)
}
defer fs.Recycle()
select {
case <-reqContext.Done():

16
pkg/request/request.go
View file

@ -1,8 +1,10 @@
package request
import (
"fmt"
"github.com/HFO4/cloudreve/pkg/auth"
"io"
"io/ioutil"
"net/http"
"time"
)
@ -104,3 +106,17 @@ func (c HTTPClient) Request(method, target string, body io.Reader, opts ...Optio
return Response{Err: nil, Response: resp}
}
// GetResponse 检查响应并获取响应正文
// todo 测试
func (resp Response) GetResponse(expectStatus int) (string, error) {
if resp.Err != nil {
return "", resp.Err
}
respBody, err := ioutil.ReadAll(resp.Response.Body)
if resp.Response.StatusCode != expectStatus {
return string(respBody),
fmt.Errorf("服务器返回非正常HTTP状态%d", resp.Response.StatusCode)
}
return string(respBody), err
}

15
pkg/request/callback.go → pkg/request/slave.go
View file

@ -8,7 +8,6 @@ import (
"github.com/HFO4/cloudreve/pkg/conf"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/HFO4/cloudreve/pkg/util"
"io/ioutil"
"time"
)
@ -36,20 +35,14 @@ func RemoteCallback(url string, body serializer.RemoteUploadCallback) error {
}
// 检查返回HTTP状态码
if resp.Response.StatusCode != 200 {
util.Log().Debug("服务端返回非正常状态码：%d", resp.Response.StatusCode)
return serializer.NewError(serializer.CodeCallbackError, "服务端返回非正常状态码", nil)
}
// 检查返回API状态码
var response serializer.Response
rawResp, err := ioutil.ReadAll(resp.Response.Body)
rawResp, err := resp.GetResponse(200)
if err != nil {
return serializer.NewError(serializer.CodeCallbackError, "无法读取响应正文", err)
return serializer.NewError(serializer.CodeCallbackError, "服务器返回异常响应", err)
}
// 解析回调服务端响应
err = json.Unmarshal(rawResp, &response)
var response serializer.Response
err = json.Unmarshal([]byte(rawResp), &response)
if err != nil {
util.Log().Debug("无法解析回调服务端响应：%s", string(rawResp))
return serializer.NewError(serializer.CodeCallbackError, "无法解析服务端返回的响应", err)

0
pkg/request/callback_test.go → pkg/request/slave_test.go
View file

6
pkg/serializer/slave.go Normal file
View file

@ -0,0 +1,6 @@
package serializer
// RemoteDeleteRequest 远程策略删除接口请求正文
type RemoteDeleteRequest struct {
Files []string `json:"files"`
}

1
pkg/serializer/upload.go
View file

@ -59,7 +59,6 @@ func DecodeUploadPolicy(raw string) (*UploadPolicy, error) {
}
// EncodeUploadPolicy 序列化Header中携带的上传策略
// TODO 测试
func (policy *UploadPolicy) EncodeUploadPolicy() (string, error) {
jsonRes, err := json.Marshal(policy)
if err != nil {

1
routers/controllers/file.go
View file

@ -265,7 +265,6 @@ func FileUploadStream(c *gin.Context) {
c.JSON(200, serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err))
return
}
defer fs.Recycle()
// 给文件系统分配钩子
fs.Use("BeforeUpload", filesystem.HookValidateFile)

16
routers/controllers/slave.go
View file

@ -25,7 +25,6 @@ func SlaveUpload(c *gin.Context) {
c.JSON(200, serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err))
return
}
defer fs.Recycle()
fs.Handler = local.Handler{}
// 从请求中取得上传策略
@ -115,3 +114,18 @@ func SlavePreview(c *gin.Context) {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveDelete 从机删除
func SlaveDelete(c *gin.Context) {
// 创建上下文
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var service explorer.SlaveFilesService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.Delete(ctx, c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}

2
routers/router.go
View file

@ -39,6 +39,8 @@ func InitSlaveRouter() *gin.Engine {
v3.GET("download/:speed/:path/:name", controllers.SlaveDownload)
// 预览 / 外链
v3.GET("source/:speed/:path/:name", controllers.SlavePreview)
// 删除文件
v3.POST("delete", controllers.SlaveDelete)
}
return r
}

35
service/explorer/file.go
View file

@ -3,6 +3,8 @@ package explorer
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/filesystem"
@ -41,6 +43,11 @@ type SlaveDownloadService struct {
Speed int `uri:"speed" binding:"min=0"`
}
// SlaveFilesService 从机多文件相关服务
type SlaveFilesService struct {
Files []string `json:"files" binding:"required,gt=0"`
}
// DownloadArchived 下載已打包的多文件
func (service *DownloadService) DownloadArchived(ctx context.Context, c *gin.Context) serializer.Response {
// 创建文件系统
@ -255,7 +262,6 @@ func (service *SingleFileService) PutContent(ctx context.Context, c *gin.Context
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 取得现有文件
exist, originFile := fs.IsFileExist(service.Path)
@ -288,7 +294,7 @@ func (service *SingleFileService) PutContent(ctx context.Context, c *gin.Context
}
}
// ServeFile 通过签名URL的文件下载从机文件
// ServeFile 通过签名的URL下载从机文件
func (service *SlaveDownloadService) ServeFile(ctx context.Context, c *gin.Context, isDownload bool) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewAnonymousFileSystem()
@ -337,3 +343,28 @@ func (service *SlaveDownloadService) ServeFile(ctx context.Context, c *gin.Conte
Code: 0,
}
}
// Delete 通过签名的URL删除从机文件
func (service *SlaveFilesService) Delete(ctx context.Context, c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewAnonymousFileSystem()
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 删除文件
failed, err := fs.Handler.Delete(ctx, service.Files)
if err != nil {
// 将Data字段写为字符串方便主控端解析
data, _ := json.Marshal(serializer.RemoteDeleteRequest{Files: failed})
return serializer.Response{
Code: serializer.CodeNotFullySuccess,
Data: string(data),
Msg: fmt.Sprintf("有 %d 个文件未能成功删除", len(failed)),
Error: err.Error(),
}
}
return serializer.Response{Code: 0}
}