Feat: improve thumbnails proformance and GC for local policy (#1044)
* thumb generating improvement Replace "github.com/nfnt/resize" with "golang.org/x/image/draw". Add thumb task queue to avoid oom when batch thumb operation * thumb improvement * Add some tests for thumbnail generation
This commit is contained in:
kikoqiu
GitHub
parent
4d7b8685b9
commit
54ed7e43ca
9 changed files with 160 additions and 15 deletions
3
go.mod
3
go.mod
|
|
@ -38,7 +38,8 @@ require (
|
|||
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac
|
||||
github.com/upyun/go-sdk v2.1.0+incompatible
|
||||
golang.org/x/text v0.3.2
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
|
||||
golang.org/x/text v0.3.6
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
||||
gopkg.in/go-playground/validator.v9 v9.29.1
|
||||
gopkg.in/ini.v1 v1.51.0 // indirect
|
||||
|
|
|
|||
5
go.sum
5
go.sum
|
|
@ -12,6 +12,7 @@ github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7I
|
|||
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.0.0/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible h1:A3oZlWPD/Poa19FvNbw+Zu4yKAurDBTjlRDilYGBiS4=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
|
|
@ -248,6 +249,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U
|
|||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/image v0.0.0-20190501045829-6d32002ffd75 h1:TbGuee8sSq15Iguxu4deQ7+Bqq/d2rsQejGcEtADAMQ=
|
||||
golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
|
|
@ -288,6 +291,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
|||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
|
|
|||
|
|
@ -70,9 +70,13 @@ type redis struct {
|
|||
|
||||
// 缩略图 配置
|
||||
type thumb struct {
|
||||
MaxWidth uint
|
||||
MaxHeight uint
|
||||
FileSuffix string `validate:"min=1"`
|
||||
MaxWidth uint
|
||||
MaxHeight uint
|
||||
FileSuffix string `validate:"min=1"`
|
||||
MaxTaskCount int
|
||||
EncodeMethod string `validate:"eq=jpg|eq=png"`
|
||||
EncodeQuality int `validate:"gte=1,lte=100"`
|
||||
GCAfterGen bool
|
||||
}
|
||||
|
||||
// 跨域配置
|
||||
|
|
|
|||
|
|
@ -51,9 +51,13 @@ var CORSConfig = &cors{
|
|||
|
||||
// ThumbConfig 缩略图配置
|
||||
var ThumbConfig = &thumb{
|
||||
MaxWidth: 400,
|
||||
MaxHeight: 300,
|
||||
FileSuffix: "._thumb",
|
||||
MaxWidth: 400,
|
||||
MaxHeight: 300,
|
||||
FileSuffix: "._thumb",
|
||||
MaxTaskCount: -1,
|
||||
EncodeMethod: "jpg",
|
||||
GCAfterGen: false,
|
||||
EncodeQuality: 85,
|
||||
}
|
||||
|
||||
// SlaveConfig 从机配置
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"runtime"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
|
|
@ -35,18 +38,53 @@ func (fs *FileSystem) GetThumb(ctx context.Context, id uint) (*response.ContentR
|
|||
ctx = context.WithValue(ctx, fsctx.ThumbSizeCtx, [2]uint{w, h})
|
||||
ctx = context.WithValue(ctx, fsctx.FileModelCtx, fs.FileTarget[0])
|
||||
res, err := fs.Handler.Thumb(ctx, fs.FileTarget[0].SourceName)
|
||||
if err == nil && conf.SystemConfig.Mode == "master" {
|
||||
res.MaxAge = model.GetIntSetting("preview_timeout", 60)
|
||||
}
|
||||
|
||||
// 本地存储策略出错时重新生成缩略图
|
||||
if err != nil && fs.Policy.Type == "local" {
|
||||
fs.GenerateThumbnail(ctx, &fs.FileTarget[0])
|
||||
res, err = fs.Handler.Thumb(ctx, fs.FileTarget[0].SourceName)
|
||||
}
|
||||
|
||||
if err == nil && conf.SystemConfig.Mode == "master" {
|
||||
res.MaxAge = model.GetIntSetting("preview_timeout", 60)
|
||||
}
|
||||
|
||||
return res, err
|
||||
}
|
||||
|
||||
// thumbPool 要使用的任务池
|
||||
var thumbPool *Pool
|
||||
var once sync.Once
|
||||
|
||||
// Pool 带有最大配额的任务池
|
||||
type Pool struct {
|
||||
// 容量
|
||||
worker chan int
|
||||
}
|
||||
|
||||
// Init 初始化任务池
|
||||
func getThumbWorker() *Pool {
|
||||
once.Do(func() {
|
||||
maxWorker := conf.ThumbConfig.MaxTaskCount
|
||||
if maxWorker <= 0 {
|
||||
maxWorker = runtime.GOMAXPROCS(0)
|
||||
}
|
||||
thumbPool = &Pool{
|
||||
worker: make(chan int, maxWorker),
|
||||
}
|
||||
util.Log().Debug("初始化Thumb任务队列，WorkerNum = %d", maxWorker)
|
||||
})
|
||||
return thumbPool
|
||||
}
|
||||
func (pool *Pool) addWorker() {
|
||||
pool.worker <- 1
|
||||
util.Log().Debug("Thumb任务队列，addWorker")
|
||||
}
|
||||
func (pool *Pool) releaseWorker() {
|
||||
util.Log().Debug("Thumb任务队列，releaseWorker")
|
||||
<-pool.worker
|
||||
}
|
||||
|
||||
// GenerateThumbnail 尝试为本地策略文件生成缩略图并获取图像原始大小
|
||||
// TODO 失败时，如果之前还有图像信息，则清除
|
||||
func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
|
||||
|
|
@ -65,6 +103,8 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
|
|||
return
|
||||
}
|
||||
defer source.Close()
|
||||
getThumbWorker().addWorker()
|
||||
defer getThumbWorker().releaseWorker()
|
||||
|
||||
image, err := thumb.NewThumbFromFile(source, file.Name)
|
||||
if err != nil {
|
||||
|
|
@ -79,6 +119,12 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
|
|||
image.GetThumb(fs.GenerateThumbnailSize(w, h))
|
||||
// 保存到文件
|
||||
err = image.Save(util.RelativePath(file.SourceName + conf.ThumbConfig.FileSuffix))
|
||||
image = nil
|
||||
if conf.ThumbConfig.GCAfterGen {
|
||||
util.Log().Debug("GenerateThumbnail runtime.GC")
|
||||
runtime.GC()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
util.Log().Warning("无法保存缩略图：%s", err)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -38,3 +38,12 @@ func TestFileSystem_GetThumb(t *testing.T) {
|
|||
asserts.EqualValues(50, res.MaxAge)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSystem_ThumbWorker(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
|
||||
asserts.NotPanics(func() {
|
||||
getThumbWorker().addWorker()
|
||||
getThumbWorker().releaseWorker()
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,9 +12,11 @@ import (
|
|||
"strings"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
|
||||
"github.com/nfnt/resize"
|
||||
//"github.com/nfnt/resize"
|
||||
"golang.org/x/image/draw"
|
||||
)
|
||||
|
||||
// Thumb 缩略图
|
||||
|
|
@ -58,7 +60,8 @@ func NewThumbFromFile(file io.Reader, name string) (*Thumb, error) {
|
|||
|
||||
// GetThumb 生成给定最大尺寸的缩略图
|
||||
func (image *Thumb) GetThumb(width, height uint) {
|
||||
image.src = resize.Thumbnail(width, height, image.src, resize.Lanczos3)
|
||||
//image.src = resize.Thumbnail(width, height, image.src, resize.Lanczos3)
|
||||
image.src = Thumbnail(width, height, image.src)
|
||||
}
|
||||
|
||||
// GetSize 获取图像尺寸
|
||||
|
|
@ -75,12 +78,59 @@ func (image *Thumb) Save(path string) (err error) {
|
|||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
switch conf.ThumbConfig.EncodeMethod {
|
||||
case "png":
|
||||
err = png.Encode(out, image.src)
|
||||
default:
|
||||
err = jpeg.Encode(out, image.src, &jpeg.Options{Quality: conf.ThumbConfig.EncodeQuality})
|
||||
}
|
||||
|
||||
err = png.Encode(out, image.src)
|
||||
return err
|
||||
|
||||
}
|
||||
|
||||
// Thumbnail will downscale provided image to max width and height preserving
|
||||
// original aspect ratio and using the interpolation function interp.
|
||||
// It will return original image, without processing it, if original sizes
|
||||
// are already smaller than provided constraints.
|
||||
func Thumbnail(maxWidth, maxHeight uint, img image.Image) image.Image {
|
||||
origBounds := img.Bounds()
|
||||
origWidth := uint(origBounds.Dx())
|
||||
origHeight := uint(origBounds.Dy())
|
||||
newWidth, newHeight := origWidth, origHeight
|
||||
|
||||
// Return original image if it have same or smaller size as constraints
|
||||
if maxWidth >= origWidth && maxHeight >= origHeight {
|
||||
return img
|
||||
}
|
||||
|
||||
// Preserve aspect ratio
|
||||
if origWidth > maxWidth {
|
||||
newHeight = uint(origHeight * maxWidth / origWidth)
|
||||
if newHeight < 1 {
|
||||
newHeight = 1
|
||||
}
|
||||
newWidth = maxWidth
|
||||
}
|
||||
|
||||
if newHeight > maxHeight {
|
||||
newWidth = uint(newWidth * maxHeight / newHeight)
|
||||
if newWidth < 1 {
|
||||
newWidth = 1
|
||||
}
|
||||
newHeight = maxHeight
|
||||
}
|
||||
return Resize(newWidth, newHeight, img)
|
||||
}
|
||||
|
||||
func Resize(newWidth, newHeight uint, img image.Image) image.Image {
|
||||
// Set the expected size that you want:
|
||||
dst := image.NewRGBA(image.Rect(0, 0, int(newWidth), int(newHeight)))
|
||||
// Resize:
|
||||
draw.BiLinear.Scale(dst, dst.Rect, img, img.Bounds(), draw.Src, nil)
|
||||
return dst
|
||||
}
|
||||
|
||||
// CreateAvatar 创建头像
|
||||
func (image *Thumb) CreateAvatar(uid uint) error {
|
||||
// 读取头像相关设定
|
||||
|
|
@ -92,7 +142,8 @@ func (image *Thumb) CreateAvatar(uid uint) error {
|
|||
// 生成头像缩略图
|
||||
src := image.src
|
||||
for k, size := range []int{s, m, l} {
|
||||
image.src = resize.Resize(uint(size), uint(size), src, resize.Lanczos3)
|
||||
//image.src = resize.Resize(uint(size), uint(size), src, resize.Lanczos3)
|
||||
image.src = Resize(uint(size), uint(size), src)
|
||||
err := image.Save(filepath.Join(savePath, fmt.Sprintf("avatar_%d_%d.png", uid, k)))
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -86,6 +86,30 @@ func TestThumb_GetThumb(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestThumb_Thumbnail(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
{
|
||||
img := image.NewRGBA(image.Rect(0, 0, 500, 200))
|
||||
thumb := Thumbnail(100, 100, img)
|
||||
asserts.Equal(thumb.Bounds(), image.Rect(0, 0, 100, 40))
|
||||
}
|
||||
{
|
||||
img := image.NewRGBA(image.Rect(0, 0, 200, 200))
|
||||
thumb := Thumbnail(100, 100, img)
|
||||
asserts.Equal(thumb.Bounds(), image.Rect(0, 0, 100, 100))
|
||||
}
|
||||
{
|
||||
img := image.NewRGBA(image.Rect(0, 0, 500, 500))
|
||||
thumb := Thumbnail(100, 100, img)
|
||||
asserts.Equal(thumb.Bounds(), image.Rect(0, 0, 100, 100))
|
||||
}
|
||||
{
|
||||
img := image.NewRGBA(image.Rect(0, 0, 200, 500))
|
||||
thumb := Thumbnail(100, 100, img)
|
||||
asserts.Equal(thumb.Bounds(), image.Rect(0, 0, 40, 100))
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumb_Save(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
file := CreateTestImage()
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"sync"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
|
|
@ -180,7 +181,7 @@ func Thumb(c *gin.Context) {
|
|||
}
|
||||
|
||||
defer resp.Content.Close()
|
||||
http.ServeContent(c.Writer, c.Request, "thumb.png", fs.FileTarget[0].UpdatedAt, resp.Content)
|
||||
http.ServeContent(c.Writer, c.Request, "thumb."+conf.ThumbConfig.EncodeMethod, fs.FileTarget[0].UpdatedAt, resp.Content)
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue