Infrastructure/Cloudreve
0
Fork 0
Code Issues Pull requests Activity

feat(thumb): generate and return sidecar thumb

Browse source
This commit is contained in:
Aaron Liu 2023-04-07 19:26:39 +08:00
parent 7cb5e68b78
commit 62b73b577b
12 changed files with 130 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
bootstrap/init.go
View file

@ -39,13 +39,7 @@ func Init(path string, statics fs.FS) {
{
"both",
func() {
cache.Init(conf.SystemConfig.Mode == "slave")
},
},
{
"master",
func() {
model.Init()
cache.Init()
},
},
{
@ -54,6 +48,18 @@ func Init(path string, statics fs.FS) {
model.InitSlaveDefaults()
},
},
{
"slave",
func() {
cache.InitSlaveOverwrites()
},
},
{
"master",
func() {
model.Init()
},
},
{
"both",
func() {

6
go.mod
View file

@ -119,6 +119,7 @@ require (
github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/samber/lo v1.38.1 // indirect
github.com/satori/go.uuid v1.2.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/soheilhy/cmux v0.1.5 // indirect
@ -146,13 +147,14 @@ require (
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.16.0 // indirect
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
golang.org/x/mod v0.4.2 // indirect
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
golang.org/x/net v0.0.0-20220630215102-69896b714898 // indirect
golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.4.0 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.0 // indirect
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210510173355-fb37daa5cd7a // indirect

6
go.sum
View file

@ -776,6 +776,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
@ -988,6 +990,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM=
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@ -1018,6 +1022,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -1254,6 +1259,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

2
models/defaults.go
View file

@ -112,6 +112,8 @@ Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; verti
{Name: "thumb_ffmpeg_enabled", Value: "0", Type: "thumb"},
{Name: "thumb_vips_path", Value: "vips", Type: "thumb"},
{Name: "thumb_ffmpeg_path", Value: "ffmpeg", Type: "thumb"},
{Name: "thumb_proxy_enabled", Value: "0", Type: "thumb"},
{Name: "thumb_proxy_policy", Value: "[]", Type: "thumb"},
{Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"},
{Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"},
{Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"},

5
models/file.go
View file

@ -466,3 +466,8 @@ func (file *File) GetPosition() string {
func (file *File) ShouldLoadThumb() bool {
return file.MetadataSerialized[ThumbStatusMetadataKey] != ThumbStatusNotAvailable
}
// return sidecar thumb file name
func (file *File) ThumbFile() string {
return file.SourceName + GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")
}

12
models/policy.go
View file

@ -4,6 +4,7 @@ import (
"encoding/gob"
"encoding/json"
"github.com/gofrs/uuid"
"github.com/samber/lo"
"path"
"path/filepath"
"strconv"
@ -227,3 +228,14 @@ func (policy *Policy) UpdateAccessKeyAndClearCache(s string) error {
func (policy *Policy) ClearCache() {
cache.Deletes([]string{strconv.FormatUint(uint64(policy.ID), 10)}, "policy_")
}
// CouldProxyThumb return if proxy thumbs is allowed for this policy.
func (policy *Policy) CouldProxyThumb() bool {
if policy.Type == "local" || !IsTrueVal(GetSettingByName("thumb_proxy_enabled")) {
return false
}
allowed := make([]uint, 0)
_ = json.Unmarshal([]byte(GetSettingByName("thumb_proxy_policy")), &allowed)
return lo.Contains[uint](allowed, policy.ID)
}

12
pkg/cache/driver.go vendored
View file

@ -10,7 +10,7 @@ import (
var Store Driver = NewMemoStore()
// Init 初始化缓存
func Init(isSlave bool) {
func Init() {
if conf.RedisConfig.Server != "" && gin.Mode() != gin.TestMode {
Store = NewRedisStore(
10,
@ -20,12 +20,12 @@ func Init(isSlave bool) {
conf.RedisConfig.DB,
)
}
}
if isSlave {
err := Store.Sets(conf.OptionOverwrite, "setting_")
if err != nil {
util.Log().Warning("Failed to overwrite database setting: %s", err)
}
func InitSlaveOverwrites() {
err := Store.Sets(conf.OptionOverwrite, "setting_")
if err != nil {
util.Log().Warning("Failed to overwrite database setting: %s", err)
}
}

2
pkg/filesystem/driver/local/handler.go
View file

@ -203,7 +203,7 @@ func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.Co
return nil, driver.ErrorThumbNotExist
}
thumbFile, err := handler.Get(ctx, file.SourceName+model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"))
thumbFile, err := handler.Get(ctx, file.ThumbFile())
if err != nil {
if errors.Is(err, os.ErrNotExist) {
err = fmt.Errorf("thumb not exist: %w (%w)", err, driver.ErrorThumbNotExist)

2
pkg/filesystem/driver/remote/handler.go
View file

@ -208,7 +208,7 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// quick check by extensions
// quick check by extension name
supported := []string{"png", "jpg", "jpeg", "gif"}
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
supported = handler.Policy.OptionsSerialized.ThumbExts

81
pkg/filesystem/image.go
View file

@ -3,7 +3,7 @@ package filesystem
import (
"context"
"errors"
"io"
"os"
"sync"
"runtime"
@ -32,17 +32,42 @@ func (fs *FileSystem) GetThumb(ctx context.Context, id uint) (*response.ContentR
}, ErrObjectNotExist
}
file := fs.FileTarget[0]
w, h := fs.GenerateThumbnailSize(0, 0)
ctx = context.WithValue(ctx, fsctx.ThumbSizeCtx, [2]uint{w, h})
ctx = context.WithValue(ctx, fsctx.FileModelCtx, fs.FileTarget[0])
res, err := fs.Handler.Thumb(ctx, &fs.FileTarget[0])
ctx = context.WithValue(ctx, fsctx.FileModelCtx, file)
res, err := fs.Handler.Thumb(ctx, &file)
if errors.Is(err, driver.ErrorThumbNotExist) {
// Regenerate thumb if the thumb is not initialized yet
fs.GenerateThumbnail(ctx, &fs.FileTarget[0])
res, err = fs.Handler.Thumb(ctx, &fs.FileTarget[0])
fs.GenerateThumbnail(ctx, &file)
res, err = fs.Handler.Thumb(ctx, &file)
} else if errors.Is(err, driver.ErrorThumbNotSupported) {
// Policy handler explicitly indicates thumb not available
_ = updateThumbStatus(&fs.FileTarget[0], model.ThumbStatusNotAvailable)
// Policy handler explicitly indicates thumb not available, check if proxy is enabled
if fs.Policy.CouldProxyThumb() {
// if thumb id marked as existed, redirect to "sidecar" thumb file.
if file.MetadataSerialized != nil &&
file.MetadataSerialized[model.ThumbStatusMetadataKey] == model.ThumbStatusExist {
// redirect to sidecar file
res = &response.ContentResponse{
Redirect: true,
}
res.URL, err = fs.Handler.Source(
ctx,
file.ThumbFile(),
*model.GetSiteURL(),
int64(model.GetIntSetting("preview_timeout", 60)),
false,
0,
)
} else {
// if not exist, generate and upload the sidecar thumb.
fs.GenerateThumbnail(ctx, &file)
res, err = fs.Handler.Thumb(ctx, &file)
}
} else {
// thumb not supported and proxy is disabled, mark as not available
_ = updateThumbStatus(&file, model.ThumbStatusNotAvailable)
}
}
if err == nil && conf.SystemConfig.Mode == "master" {
@ -100,20 +125,7 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
getThumbWorker().addWorker()
defer getThumbWorker().releaseWorker()
r, w := io.Pipe()
defer w.Close()
errChan := make(chan error, 1)
go func(errChan chan error) {
errChan <- fs.Handler.Put(newCtx, &fsctx.FileStream{
Mode: fsctx.Overwrite,
File: io.NopCloser(r),
Seeker: nil,
SavePath: file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"),
})
}(errChan)
if err = thumb.Generators.Generate(source, w, file.Name, model.GetSettingByNames(
thumbPath, err := thumb.Generators.Generate(source, file.Name, model.GetSettingByNames(
"thumb_width",
"thumb_height",
"thumb_builtin_enabled",
@ -121,16 +133,33 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
"thumb_ffmpeg_enabled",
"thumb_vips_path",
"thumb_ffmpeg_path",
)); err != nil {
))
if err != nil {
util.Log().Warning("Failed to generate thumb for %s: %s", file.Name, err)
_ = updateThumbStatus(file, model.ThumbStatusNotAvailable)
w.Close()
<-errChan
return
}
w.Close()
if err = <-errChan; err != nil {
thumbFile, err := os.Open(thumbPath)
if err != nil {
util.Log().Warning("Failed to open temp thumb %q: %s", thumbFile, err)
return
}
defer thumbFile.Close()
fileInfo, err := thumbFile.Stat()
if err != nil {
util.Log().Warning("Failed to stat temp thumb %q: %s", thumbFile, err)
return
}
if err = fs.Handler.Put(newCtx, &fsctx.FileStream{
Mode: fsctx.Overwrite,
File: thumbFile,
Seeker: thumbFile,
Size: uint64(fileInfo.Size()),
SavePath: file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"),
}); err != nil {
util.Log().Warning("Failed to save thumb for %s: %s", file.Name, err)
return
}

24
pkg/thumb/builtin.go
View file

@ -12,7 +12,7 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gofrs/uuid"
//"github.com/nfnt/resize"
"golang.org/x/image/draw"
)
@ -156,14 +156,30 @@ func (image *Thumb) CreateAvatar(uid uint) error {
type Builtin struct{}
func (b Builtin) Generate(file io.Reader, w io.Writer, name string, options map[string]string) error {
func (b Builtin) Generate(file io.Reader, name string, options map[string]string) (string, error) {
img, err := NewThumbFromFile(file, name)
if err != nil {
return err
return "", err
}
img.GetThumb(thumbSize(options))
return img.Save(w)
tempPath := filepath.Join(
util.RelativePath(model.GetSettingByName("temp_path")),
"thumb",
fmt.Sprintf("thumb_%s", uuid.Must(uuid.NewV4()).String()),
)
thumbFile, err := util.CreatNestedFile(tempPath)
if err != nil {
return "", fmt.Errorf("failed to create temp file: %w", err)
}
defer thumbFile.Close()
if err := img.Save(thumbFile); err != nil {
return "", err
}
return tempPath, nil
}
func (b Builtin) Priority() int {

10
pkg/thumb/pipeline.go
View file

@ -12,7 +12,7 @@ import (
// Generator generates a thumbnail for a given reader.
type Generator interface {
Generate(file io.Reader, w io.Writer, name string, options map[string]string) error
Generate(file io.Reader, name string, options map[string]string) (string, error)
// Priority of execution order, smaller value means higher priority.
Priority() int
@ -51,19 +51,19 @@ func RegisterGenerator(generator Generator) {
sort.Sort(Generators)
}
func (p GeneratorList) Generate(file io.Reader, w io.Writer, name string, options map[string]string) error {
func (p GeneratorList) Generate(file io.Reader, name string, options map[string]string) (string, error) {
for _, generator := range p {
if model.IsTrueVal(options[generator.EnableFlag()]) {
err := generator.Generate(file, w, name, options)
res, err := generator.Generate(file, name, options)
if errors.Is(err, ErrPassThrough) {
util.Log().Debug("Failed to generate thumbnail for %s: %s, passing through to next generator.", name, err)
continue
}
return err
return res, err
}
}
return ErrNotAvailable
return "", ErrNotAvailable
}
func (p GeneratorList) Priority() int {