From 52c2422be95ae3e1d39f6fe9ed0998297e208fe1 Mon Sep 17 00:00:00 2001
From: HFO4 <912394456@qq.com>
Date: Tue, 28 Apr 2020 11:43:32 +0800
Subject: [PATCH] Feat: list oss files

---
 pkg/filesystem/driver/oss/handler.go | 76 ++++++++++++++++++++--------
 pkg/task/import.go                   | 17 ++++---
 pkg/task/import_test.go              |  2 +
 3 files changed, 68 insertions(+), 27 deletions(-)

diff --git a/pkg/filesystem/driver/oss/handler.go b/pkg/filesystem/driver/oss/handler.go
index da00fc0..50dd557 100644
--- a/pkg/filesystem/driver/oss/handler.go
+++ b/pkg/filesystem/driver/oss/handler.go
@@ -18,6 +18,7 @@ import (
 	"io"
 	"net/url"
 	"path"
+	"path/filepath"
 	"strings"
 	"time"
 )
@@ -100,39 +101,72 @@ func (handler *Driver) InitOSSClient() error {
 	return nil
 }
 
-func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
+// List 列出OSS上的文件
+func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
 	// 初始化客户端
 	if err := handler.InitOSSClient(); err != nil {
 		return nil, err
 	}
 
 	// 列取文件
-	options := []oss.Option{
-		oss.Prefix(strings.TrimPrefix(path, "/")),
+	base = strings.TrimPrefix(base, "/")
+	delimiter := ""
+	if !recursive {
+		delimiter = "/"
 	}
-	if recursive {
-		options = append(options, oss.Delimiter("/"))
-	}
-	listRes, err := handler.bucket.ListObjects(options...)
-	if err != nil {
-		return nil, err
+	marker := ""
+	var (
+		objects []oss.ObjectProperties
+		commons []string
+	)
+
+	for {
+		subRes, err := handler.bucket.ListObjects(oss.Marker(marker), oss.Prefix(base),
+			oss.MaxKeys(1000), oss.Delimiter(delimiter))
+		if err != nil {
+			return nil, err
+		}
+		objects = append(objects, subRes.Objects...)
+		commons = append(commons, subRes.CommonPrefixes...)
+		marker = subRes.NextMarker
+		if marker == "" {
+			break
+		}
 	}
 
 	// 处理列取结果
-	res := make([]response.Object, len(listRes.CommonPrefixes)+len(listRes.Objects))
+	res := make([]response.Object, 0, len(objects)+len(commons))
 	// 处理目录
-	//for _,object := range listRes.CommonPrefixes{
-	//	res = append(res,response.Object{
-	//		Name:         "",
-	//		RelativePath: "",
-	//		Source:       "",
-	//		Size:         0,
-	//		IsDir:        false,
-	//		LastModify:   time.Time{},
-	//	})
-	//}
+	for _, object := range commons {
+		rel, err := filepath.Rel(base, object)
+		if err != nil {
+			continue
+		}
+		res = append(res, response.Object{
+			Name:         path.Base(object),
+			RelativePath: filepath.ToSlash(rel),
+			Size:         0,
+			IsDir:        true,
+			LastModify:   time.Now(),
+		})
+	}
+	// 处理文件
+	for _, object := range objects {
+		rel, err := filepath.Rel(base, object.Key)
+		if err != nil {
+			continue
+		}
+		res = append(res, response.Object{
+			Name:         path.Base(object.Key),
+			Source:       object.Key,
+			RelativePath: filepath.ToSlash(rel),
+			Size:         uint64(object.Size),
+			IsDir:        false,
+			LastModify:   object.LastModified,
+		})
+	}
 
-	return res, err
+	return res, nil
 }
 
 // Get 获取文件
diff --git a/pkg/task/import.go b/pkg/task/import.go
index bc08c42..21684d6 100644
--- a/pkg/task/import.go
+++ b/pkg/task/import.go
@@ -146,13 +146,18 @@ func (job *ImportTask) Do() {
 			if parent, ok := pathCache[virtualPath]; ok {
 				parentFolder = parent
 			} else {
-				folder, err := fs.CreateDirectory(context.Background(), virtualPath)
-				if err != nil {
-					util.Log().Warning("导入任务无法创建用户目录[%s], %s",
-						virtualPath, err)
-					continue
+				exist, folder := fs.IsPathExist(virtualPath)
+				if exist {
+					parentFolder = folder
+				} else {
+					folder, err := fs.CreateDirectory(context.Background(), virtualPath)
+					if err != nil {
+						util.Log().Warning("导入任务无法创建用户目录[%s], %s",
+							virtualPath, err)
+						continue
+					}
+					parentFolder = folder
 				}
-				parentFolder = folder
 			}
 
 			// 插入文件记录
diff --git a/pkg/task/import_test.go b/pkg/task/import_test.go
index d6aaafb..21d1d3c 100644
--- a/pkg/task/import_test.go
+++ b/pkg/task/import_test.go
@@ -142,6 +142,8 @@ func TestImportTask_Do(t *testing.T) {
 		mock.ExpectCommit()
 		// 查找父目录，但是不存在
 		mock.ExpectQuery("SELECT(.+)folders").WillReturnRows(sqlmock.NewRows([]string{"id"}))
+		// 仍然不存在
+		mock.ExpectQuery("SELECT(.+)folders").WillReturnRows(sqlmock.NewRows([]string{"id"}))
 		// 创建文件时查找父目录，仍然不存在
 		mock.ExpectQuery("SELECT(.+)folders").WillReturnRows(sqlmock.NewRows([]string{"id"}))