From 7a3d44451b46696de96b1369471db8712e8dc12f Mon Sep 17 00:00:00 2001 From: Weidi Deng Date: Wed, 20 Apr 2022 19:56:00 +0800 Subject: [PATCH 1/5] precompress embedded frontend. import mholt/archiver. --- Dockerfile | 3 ++- bootstrap/init.go | 4 ++-- bootstrap/static.go | 5 ++--- build.sh | 2 ++ go.mod | 10 ++++++++++ go.sum | 25 +++++++++++++++++++++++++ main.go | 22 +++++++++++++++++----- 7 files changed, 60 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 028fb8c..8c82570 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17-alpine as cloudreve_builder # install dependencies and build tools -RUN apk update && apk add --no-cache wget curl git yarn build-base gcc abuild binutils binutils-doc gcc-doc +RUN apk update && apk add --no-cache wget curl git yarn build-base gcc abuild binutils binutils-doc gcc-doc zip WORKDIR /cloudreve_builder RUN git clone --recurse-submodules https://github.com/cloudreve/Cloudreve.git @@ -14,6 +14,7 @@ RUN yarn run build && rm -rf build/*.map # build backend WORKDIR /cloudreve_builder/Cloudreve +RUN zip -r assets.zip assets RUN tag_name=$(git describe --tags) \ && export COMMIT_SHA=$(git rev-parse --short HEAD) \ && go build -a -o cloudreve -ldflags " -X 'github.com/HFO4/cloudreve/pkg/conf.BackendVersion=$tag_name' -X 'github.com/HFO4/cloudreve/pkg/conf.LastCommit=$COMMIT_SHA'" diff --git a/bootstrap/init.go b/bootstrap/init.go index ff55beb..2aea14d 100644 --- a/bootstrap/init.go +++ b/bootstrap/init.go @@ -1,7 +1,6 @@ package bootstrap import ( - "embed" model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/models/scripts" "github.com/cloudreve/Cloudreve/v3/pkg/aria2" @@ -14,10 +13,11 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/mq" "github.com/cloudreve/Cloudreve/v3/pkg/task" "github.com/gin-gonic/gin" + "io/fs" ) // Init 初始化启动 -func Init(path string, statics embed.FS) { +func Init(path string, statics fs.FS) { InitApplication() conf.Init(path) // Debug 关闭时，切换为生产模式 diff --git a/bootstrap/static.go b/bootstrap/static.go index 82397e8..21d4a04 100644 --- a/bootstrap/static.go +++ b/bootstrap/static.go @@ -2,7 +2,6 @@ package bootstrap import ( "bufio" - "embed" "encoding/json" "io" "io/fs" @@ -45,7 +44,7 @@ func (b *GinFS) Exists(prefix string, filepath string) bool { } // InitStatic 初始化静态资源文件 -func InitStatic(statics embed.FS) { +func InitStatic(statics fs.FS) { if util.Exists(util.RelativePath(StaticFolder)) { util.Log().Info("检测到 statics 目录存在，将使用此目录下的静态资源文件") StaticFS = static.LocalFile(util.RelativePath("statics"), false) @@ -96,7 +95,7 @@ func InitStatic(statics embed.FS) { } // Eject 抽离内置静态资源 -func Eject(statics embed.FS) { +func Eject(statics fs.FS) { // 初始化静态资源 embedFS, err := fs.Sub(statics, "assets/build") if err != nil { diff --git a/build.sh b/build.sh index c1ec24e..c008fb4 100755 --- a/build.sh +++ b/build.sh @@ -31,6 +31,8 @@ buildAssets() { yarn run build cd build rm -rf *.map + cd $REPO + zip -r assets.zip assets } buildBinary() { diff --git a/go.mod b/go.mod index 9f939d4..250d868 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/hashicorp/go-version v1.3.0 github.com/jinzhu/gorm v1.9.11 github.com/juju/ratelimit v1.0.1 + github.com/mholt/archiver/v4 v4.0.0-alpha.6 github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2 github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.2.0 @@ -41,6 +42,7 @@ require ( require ( cloud.google.com/go v0.37.4 // indirect github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect + github.com/andybalholm/brotli v1.0.4 // indirect github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect @@ -48,6 +50,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/dsnet/compress v0.0.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect @@ -55,6 +58,7 @@ require ( github.com/go-sql-driver/mysql v1.5.0 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.3.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/certificate-transparency-go v1.0.21 // indirect github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/securecookie v1.1.1 // indirect @@ -63,6 +67,8 @@ require ( github.com/jmespath/go-jmespath v0.3.0 // indirect github.com/json-iterator/go v1.1.9 // indirect github.com/katzenpost/core v0.0.7 // indirect + github.com/klauspost/compress v1.15.1 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/lib/pq v1.1.1 // indirect github.com/mattn/go-colorable v0.1.4 // indirect @@ -72,12 +78,16 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/mozillazg/go-httpheader v0.2.1 // indirect + github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect + github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438 // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/smartystreets/goconvey v1.6.4 // indirect github.com/stretchr/objx v0.2.0 // indirect + github.com/therootcompany/xz v1.0.1 // indirect github.com/ugorji/go/codec v1.1.7 // indirect + github.com/ulikunitz/xz v0.5.10 // indirect golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect golang.org/x/sys v0.0.0-20211020174200-9d6173849985 // indirect diff --git a/go.sum b/go.sum index e120a09..5847d29 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7I github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/aws/aws-sdk-go v1.31.5 h1:DFA7BzTydO4etqsTja+x7UfkOKQUv1xzEluLvNk81L0= github.com/aws/aws-sdk-go v1.31.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= @@ -38,6 +40,9 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 h1:tkum0XDgf github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4 h1:rXVUYM3uQcdXgSvQ5Bo+JZFMnLi0H3jib+2mz7B6M4U= github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4/go.mod h1:KR2KScxcZAWdZGOUnsPGjD3ow0cvNfv3WHXC/Xz+d9g= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -103,6 +108,8 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -156,6 +163,12 @@ github.com/katzenpost/core v0.0.7 h1:ZI4oYACe/3n2iS2XsCccGbTFdhhRJunjFW05Utwna+g github.com/katzenpost/core v0.0.7/go.mod h1:UXMLmMXlBHrhMXhWTy4DvCXqwTRLOh4DP/mR1Cm1sR8= github.com/kidstuff/mongostore v0.0.0-20181113001930-e650cd85ee4b/go.mod h1:g2nVr8KZVXJSS97Jo8pJ0jgq29P6H7dG0oplUA86MQw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -184,6 +197,8 @@ github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/memcachier/mc v2.0.1+incompatible/go.mod h1:7bkvFE61leUBvXz+yxsOnGBQSZpBSPIMUQSmmSHvuXc= +github.com/mholt/archiver/v4 v4.0.0-alpha.6 h1:3wvos9Kn1GpKNBz+MpozinGREPslLo1ds1W16vTkErQ= +github.com/mholt/archiver/v4 v4.0.0-alpha.6/go.mod h1:9PTygYq90FQBWPspdwAng6dNjYiBuTYKqmA6c15KuCo= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -197,11 +212,16 @@ github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2/go.mod h1:wAQ github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= +github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= +github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -254,11 +274,16 @@ github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible h1:dqpmYaez7VB github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ= github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA= +github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= +github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/upyun/go-sdk v2.1.0+incompatible h1:OdjXghQ/TVetWV16Pz3C1/SUpjhGBVPr+cLiqZLLyq0= github.com/upyun/go-sdk v2.1.0+incompatible/go.mod h1:eu3F5Uz4b9ZE5bE5QsCL6mgSNWRwfj0zpJ9J626HEqs= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= diff --git a/main.go b/main.go index 890a7a1..ed7a409 100644 --- a/main.go +++ b/main.go @@ -1,13 +1,18 @@ package main import ( - "embed" + _ "embed" "flag" + "io" + "io/fs" + "strings" "github.com/cloudreve/Cloudreve/v3/bootstrap" "github.com/cloudreve/Cloudreve/v3/pkg/conf" "github.com/cloudreve/Cloudreve/v3/pkg/util" "github.com/cloudreve/Cloudreve/v3/routers" + + "github.com/mholt/archiver/v4" ) var ( @@ -16,21 +21,28 @@ var ( scriptName string ) -//go:embed assets/build -var staticEmbed embed.FS +//go:embed assets.zip +var staticZip string + +var staticFS fs.FS func init() { flag.StringVar(&confPath, "c", util.RelativePath("conf.ini"), "配置文件路径") flag.BoolVar(&isEject, "eject", false, "导出内置静态资源") flag.StringVar(&scriptName, "database-script", "", "运行内置数据库助手脚本") flag.Parse() - bootstrap.Init(confPath, staticEmbed) + + staticFS = archiver.ArchiveFS{ + Stream: io.NewSectionReader(strings.NewReader(staticZip), 0, int64(len(staticZip))), + Format: archiver.Zip{}, + } + bootstrap.Init(confPath, staticFS) } func main() { if isEject { // 开始导出内置静态资源文件 - bootstrap.Eject(staticEmbed) + bootstrap.Eject(staticFS) return } From 23bd1389bc45f148ed48139f99b3940fe507c2dc Mon Sep 17 00:00:00 2001 From: Weidi Deng Date: Thu, 21 Apr 2022 16:33:10 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E4=BD=BF=E7=94=A8archiver=E5=AF=B9?= =?UTF-8?q?=E5=8E=8B=E7=BC=A9=E6=96=87=E4=BB=B6=E8=BF=9B=E8=A1=8C=E8=A7=A3?= =?UTF-8?q?=E5=8E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/filesystem/archive.go | 125 ++++++++++++++++++++---------------- service/explorer/objects.go | 16 ++++- 2 files changed, 84 insertions(+), 57 deletions(-) diff --git a/pkg/filesystem/archive.go b/pkg/filesystem/archive.go index f1bc418..400f9b6 100644 --- a/pkg/filesystem/archive.go +++ b/pkg/filesystem/archive.go @@ -2,11 +2,9 @@ package filesystem import ( "archive/zip" - "bytes" "context" "fmt" "io" - "io/ioutil" "os" "path" "path/filepath" @@ -18,8 +16,7 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/util" "github.com/gin-gonic/gin" - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/transform" + "github.com/mholt/archiver/v4" ) /* =============== @@ -206,21 +203,41 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error { } defer zipFile.Close() - _, err = io.Copy(zipFile, fileStream) + // 下载前先判断是否是可解压的格式 + format, readStream, err := archiver.Identify(fs.FileTarget[0].SourceName, fileStream) if err != nil { - util.Log().Warning("无法写入临时压缩文件 %s , %s", tempZipFilePath, err) + util.Log().Warning("无法识别文件格式 %s , %s", fs.FileTarget[0].SourceName, err) return err } - zipFile.Close() - fileStream.Close() - - // 解压缩文件 - r, err := zip.OpenReader(tempZipFilePath) - if err != nil { - return err + extractor, ok := format.(archiver.Extractor) + if !ok { + return fmt.Errorf("file not an extractor %s", fs.FileTarget[0].SourceName) + } + + // 只有zip格式可以多个文件同时上传 + var isZip bool + switch extractor.(type) { + case archiver.Zip: + extractor = archiver.Zip{TextEncoding: "gb18030"} + isZip = true + } + + // 除了zip必须下载到本地，其余的可以边下载边解压 + reader := readStream + if isZip { + _, err = io.Copy(zipFile, readStream) + if err != nil { + util.Log().Warning("无法写入临时压缩文件 %s , %s", tempZipFilePath, err) + return err + } + + fileStream.Close() + + // 设置文件偏移量 + zipFile.Seek(0, io.SeekStart) + reader = zipFile } - defer r.Close() // 重设存储策略 fs.Policy = &fs.User.Policy @@ -236,64 +253,64 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error { worker <- i } - for _, f := range r.File { - fileName := f.Name - // 处理非UTF-8编码 - if f.NonUTF8 { - i := bytes.NewReader([]byte(fileName)) - decoder := transform.NewReader(i, simplifiedchinese.GB18030.NewDecoder()) - content, _ := ioutil.ReadAll(decoder) - fileName = string(content) - } + // 上传文件函数 + uploadFunc := func(fileStream io.ReadCloser, size int64, savePath, rawPath string) { + defer func() { + if isZip { + worker <- 1 + wg.Done() + } + if err := recover(); err != nil { + util.Log().Warning("上传压缩包内文件时出错") + fmt.Println(err) + } + }() - rawPath := util.FormSlash(fileName) + err := fs.UploadFromStream(ctx, &fsctx.FileStream{ + File: fileStream, + Size: uint64(size), + Name: path.Base(savePath), + VirtualPath: path.Dir(savePath), + }, true) + fileStream.Close() + if err != nil { + util.Log().Debug("无法上传压缩包内的文件%s , %s , 跳过", rawPath, err) + } + } + + // 解压缩文件，回调函数如果出错会停止解压的下一步进行，全部return nil + err = extractor.Extract(ctx, reader, nil, func(ctx context.Context, f archiver.File) error { + rawPath := util.FormSlash(f.NameInArchive) savePath := path.Join(dst, rawPath) // 路径是否合法 if !strings.HasPrefix(savePath, util.FillSlash(path.Clean(dst))) { - return fmt.Errorf("%s: illegal file path", f.Name) + util.Log().Warning("%s: illegal file path", f.NameInArchive) + return nil } // 如果是目录 - if f.FileInfo().IsDir() { + if f.FileInfo.IsDir() { fs.CreateDirectory(ctx, savePath) - continue + return nil } // 上传文件 fileStream, err := f.Open() if err != nil { util.Log().Warning("无法打开压缩包内文件%s , %s , 跳过", rawPath, err) - continue + return nil } - select { - case <-worker: + if !isZip { + uploadFunc(fileStream, f.FileInfo.Size(), savePath, rawPath) + } else { + <-worker wg.Add(1) - go func(fileStream io.ReadCloser, size int64) { - defer func() { - worker <- 1 - wg.Done() - if err := recover(); err != nil { - util.Log().Warning("上传压缩包内文件时出错") - fmt.Println(err) - } - }() - - err = fs.UploadFromStream(ctx, &fsctx.FileStream{ - File: fileStream, - Size: uint64(size), - Name: path.Base(savePath), - VirtualPath: path.Dir(savePath), - }, true) - fileStream.Close() - if err != nil { - util.Log().Debug("无法上传压缩包内的文件%s , %s , 跳过", rawPath, err) - } - }(fileStream, f.FileInfo().Size()) + go uploadFunc(fileStream, f.FileInfo.Size(), savePath, rawPath) } - - } + return nil + }) wg.Wait() - return nil + return err } diff --git a/service/explorer/objects.go b/service/explorer/objects.go index dab5850..d5adbc4 100644 --- a/service/explorer/objects.go +++ b/service/explorer/objects.go @@ -127,9 +127,19 @@ func (service *ItemDecompressService) CreateDecompressTask(c *gin.Context) seria return serializer.Err(serializer.CodeParamErr, "文件太大", nil) } - // 必须是zip压缩包 - if !strings.HasSuffix(file.Name, ".zip") { - return serializer.Err(serializer.CodeParamErr, "只能解压 ZIP 格式的压缩文件", nil) + // 支持的压缩格式后缀 + var ( + suffixes = []string{".zip", ".gz", ".xz", ".tar", ".rar"} + matched bool + ) + for _, suffix := range suffixes { + if strings.HasSuffix(file.Name, suffix) { + matched = true + break + } + } + if !matched { + return serializer.Err(serializer.CodeParamErr, "不支持该格式的压缩文件", nil) } // 创建任务 From ac78e9db0260be9dbe08d393e1069c634044ce67 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Mon, 25 Apr 2022 16:49:21 +0800 Subject: [PATCH 3/5] add empty `assets.zip` for placeholder --- Dockerfile | 2 +- assets.zip | 0 build.sh | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 assets.zip diff --git a/Dockerfile b/Dockerfile index 8c82570..ede4bd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN yarn run build && rm -rf build/*.map # build backend WORKDIR /cloudreve_builder/Cloudreve -RUN zip -r assets.zip assets +RUN zip -r - assets >assets.zip RUN tag_name=$(git describe --tags) \ && export COMMIT_SHA=$(git rev-parse --short HEAD) \ && go build -a -o cloudreve -ldflags " -X 'github.com/HFO4/cloudreve/pkg/conf.BackendVersion=$tag_name' -X 'github.com/HFO4/cloudreve/pkg/conf.LastCommit=$COMMIT_SHA'" diff --git a/assets.zip b/assets.zip new file mode 100644 index 0000000..e69de29 diff --git a/build.sh b/build.sh index c008fb4..c68ccef 100755 --- a/build.sh +++ b/build.sh @@ -32,7 +32,7 @@ buildAssets() { cd build rm -rf *.map cd $REPO - zip -r assets.zip assets + zip -r - assets >assets.zip } buildBinary() { From cb51046305e542fa9f23664cb4f2a2731533438d Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Mon, 25 Apr 2022 17:23:42 +0800 Subject: [PATCH 4/5] test: new changes in decompress method --- pkg/filesystem/archive.go | 4 +-- pkg/filesystem/archive_test.go | 47 +++++++++++++++++----------------- pkg/task/decompress.go | 2 +- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/pkg/filesystem/archive.go b/pkg/filesystem/archive.go index 400f9b6..7e62590 100644 --- a/pkg/filesystem/archive.go +++ b/pkg/filesystem/archive.go @@ -165,7 +165,7 @@ func (fs *FileSystem) doCompress(ctx context.Context, file *model.File, folder * } // Decompress 解压缩给定压缩文件到dst目录 -func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error { +func (fs *FileSystem) Decompress(ctx context.Context, src, dst, encoding string) error { err := fs.ResetFileIfNotExist(ctx, src) if err != nil { return err @@ -219,7 +219,7 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error { var isZip bool switch extractor.(type) { case archiver.Zip: - extractor = archiver.Zip{TextEncoding: "gb18030"} + extractor = archiver.Zip{TextEncoding: encoding} isZip = true } diff --git a/pkg/filesystem/archive_test.go b/pkg/filesystem/archive_test.go index 0bbb179..07f5087 100644 --- a/pkg/filesystem/archive_test.go +++ b/pkg/filesystem/archive_test.go @@ -8,6 +8,8 @@ import ( testMock "github.com/stretchr/testify/mock" "io" "os" + "path/filepath" + "runtime" "strings" "testing" @@ -147,12 +149,24 @@ func (m MockRSC) Close() error { return nil } +var basepath string + +func init() { + _, currentFile, _, _ := runtime.Caller(0) + basepath = filepath.Dir(currentFile) +} + +func Path(rel string) string { + return filepath.Join(basepath, rel) +} + func TestFileSystem_Decompress(t *testing.T) { asserts := assert.New(t) ctx := context.Background() fs := FileSystem{ User: &model.User{Model: gorm.Model{ID: 1}}, } + os.RemoveAll(util.RelativePath("tests/decompress")) // 压缩文件不存在 { @@ -162,7 +176,7 @@ func TestFileSystem_Decompress(t *testing.T) { // 查找压缩文件，未找到 mock.ExpectQuery("SELECT(.+)files(.+)"). WillReturnRows(sqlmock.NewRows([]string{"id", "name"})) - err := fs.Decompress(ctx, "/1.zip", "/") + err := fs.Decompress(ctx, "/1.zip", "/", "") asserts.NoError(mock.ExpectationsWereMet()) asserts.Error(err) } @@ -174,7 +188,7 @@ func TestFileSystem_Decompress(t *testing.T) { testHandler := new(FileHeaderMock) testHandler.On("Get", testMock.Anything, "1.zip").Return(MockRSC{}, errors.New("error")) fs.Handler = testHandler - err := fs.Decompress(ctx, "/1.zip", "/") + err := fs.Decompress(ctx, "/1.zip", "/", "") asserts.NoError(mock.ExpectationsWereMet()) asserts.Error(err) asserts.EqualError(err, "error") @@ -188,7 +202,7 @@ func TestFileSystem_Decompress(t *testing.T) { testHandler := new(FileHeaderMock) testHandler.On("Get", testMock.Anything, "1.zip").Return(MockRSC{}, nil) fs.Handler = testHandler - err := fs.Decompress(ctx, "/1.zip", "/") + err := fs.Decompress(ctx, "/1.zip", "/", "") asserts.NoError(mock.ExpectationsWereMet()) asserts.Error(err) } @@ -201,13 +215,13 @@ func TestFileSystem_Decompress(t *testing.T) { testHandler := new(FileHeaderMock) testHandler.On("Get", testMock.Anything, "1.zip").Return(MockNopRSC("1"), nil) fs.Handler = testHandler - err := fs.Decompress(ctx, "/1.zip", "/") + err := fs.Decompress(ctx, "/1.zip", "/", "") asserts.NoError(mock.ExpectationsWereMet()) asserts.Error(err) - asserts.EqualError(err, "read error") + asserts.Contains(err.Error(), "read error") } - // 无效zip文件 + // 无法重设上传策略 { cache.Set("setting_temp_path", "tests", 0) fs.FileTarget = []model.File{{SourceName: "1.zip", Policy: model.Policy{Type: "mock"}}} @@ -215,22 +229,7 @@ func TestFileSystem_Decompress(t *testing.T) { testHandler := new(FileHeaderMock) testHandler.On("Get", testMock.Anything, "1.zip").Return(MockRSC{rs: strings.NewReader("read")}, nil) fs.Handler = testHandler - err := fs.Decompress(ctx, "/1.zip", "/") - asserts.NoError(mock.ExpectationsWereMet()) - asserts.Error(err) - asserts.EqualError(err, "zip: not a valid zip file") - } - - // 无法重设上传策略 - { - zipFile, _ := os.Open(util.RelativePath("filesystem/tests/test.zip")) - fs.FileTarget = []model.File{{SourceName: "1.zip", Policy: model.Policy{Type: "mock"}}} - fs.FileTarget[0].Policy.ID = 1 - testHandler := new(FileHeaderMock) - testHandler.On("Get", testMock.Anything, "1.zip").Return(zipFile, nil) - fs.Handler = testHandler - err := fs.Decompress(ctx, "/1.zip", "/") - zipFile.Close() + err := fs.Decompress(ctx, "/1.zip", "/", "") asserts.NoError(mock.ExpectationsWereMet()) asserts.Error(err) asserts.True(util.IsEmpty(util.RelativePath("tests/decompress"))) @@ -239,7 +238,7 @@ func TestFileSystem_Decompress(t *testing.T) { // 无法上传，容量不足 { cache.Set("setting_max_parallel_transfer", "1", 0) - zipFile, _ := os.Open(util.RelativePath("filesystem/tests/test.zip")) + zipFile, _ := os.Open(Path("tests/test.zip")) fs.FileTarget = []model.File{{SourceName: "1.zip", Policy: model.Policy{Type: "mock"}}} fs.FileTarget[0].Policy.ID = 1 fs.User.Policy.Type = "mock" @@ -247,7 +246,7 @@ func TestFileSystem_Decompress(t *testing.T) { testHandler.On("Get", testMock.Anything, "1.zip").Return(zipFile, nil) fs.Handler = testHandler - fs.Decompress(ctx, "/1.zip", "/") + fs.Decompress(ctx, "/1.zip", "/", "") zipFile.Close() diff --git a/pkg/task/decompress.go b/pkg/task/decompress.go index 2c545e9..ce00278 100644 --- a/pkg/task/decompress.go +++ b/pkg/task/decompress.go @@ -82,7 +82,7 @@ func (job *DecompressTask) Do() { job.TaskModel.SetProgress(DecompressingProgress) - err = fs.Decompress(context.Background(), job.TaskProps.Src, job.TaskProps.Dst) + err = fs.Decompress(context.Background(), job.TaskProps.Src, job.TaskProps.Dst, "") if err != nil { job.SetErrorMsg("解压缩失败", err) return From 86876a1c118121808e38943866030df811c4c6e5 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Mon, 25 Apr 2022 18:07:47 +0800 Subject: [PATCH 5/5] feat: select encoding for decompressing zip file --- pkg/task/decompress.go | 14 ++++++++------ pkg/task/decompress_test.go | 4 ++-- service/explorer/objects.go | 7 ++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/pkg/task/decompress.go b/pkg/task/decompress.go index ce00278..0db2ec5 100644 --- a/pkg/task/decompress.go +++ b/pkg/task/decompress.go @@ -20,8 +20,9 @@ type DecompressTask struct { // DecompressProps 压缩任务属性 type DecompressProps struct { - Src string `json:"src"` - Dst string `json:"dst"` + Src string `json:"src"` + Dst string `json:"dst"` + Encoding string `json:"encoding"` } // Props 获取任务属性 @@ -82,7 +83,7 @@ func (job *DecompressTask) Do() { job.TaskModel.SetProgress(DecompressingProgress) - err = fs.Decompress(context.Background(), job.TaskProps.Src, job.TaskProps.Dst, "") + err = fs.Decompress(context.Background(), job.TaskProps.Src, job.TaskProps.Dst, job.TaskProps.Encoding) if err != nil { job.SetErrorMsg("解压缩失败", err) return @@ -91,12 +92,13 @@ func (job *DecompressTask) Do() { } // NewDecompressTask 新建压缩任务 -func NewDecompressTask(user *model.User, src, dst string) (Job, error) { +func NewDecompressTask(user *model.User, src, dst, encoding string) (Job, error) { newTask := &DecompressTask{ User: user, TaskProps: DecompressProps{ - Src: src, - Dst: dst, + Src: src, + Dst: dst, + Encoding: encoding, }, } diff --git a/pkg/task/decompress_test.go b/pkg/task/decompress_test.go index 33887fc..75b7cfe 100644 --- a/pkg/task/decompress_test.go +++ b/pkg/task/decompress_test.go @@ -99,7 +99,7 @@ func TestNewDecompressTask(t *testing.T) { mock.ExpectBegin() mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() - job, err := NewDecompressTask(&model.User{}, "/", "/") + job, err := NewDecompressTask(&model.User{}, "/", "/", "utf-8") asserts.NoError(mock.ExpectationsWereMet()) asserts.NotNil(job) asserts.NoError(err) @@ -110,7 +110,7 @@ func TestNewDecompressTask(t *testing.T) { mock.ExpectBegin() mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error")) mock.ExpectRollback() - job, err := NewDecompressTask(&model.User{}, "/", "/") + job, err := NewDecompressTask(&model.User{}, "/", "/", "utf-8") asserts.NoError(mock.ExpectationsWereMet()) asserts.Nil(job) asserts.Error(err) diff --git a/service/explorer/objects.go b/service/explorer/objects.go index d5adbc4..d140c32 100644 --- a/service/explorer/objects.go +++ b/service/explorer/objects.go @@ -55,8 +55,9 @@ type ItemCompressService struct { // ItemDecompressService 文件解压缩任务服务 type ItemDecompressService struct { - Src string `json:"src"` - Dst string `json:"dst" binding:"required,min=1,max=65535"` + Src string `json:"src"` + Dst string `json:"dst" binding:"required,min=1,max=65535"` + Encoding string `json:"encoding"` } // ItemPropertyService 获取对象属性服务 @@ -143,7 +144,7 @@ func (service *ItemDecompressService) CreateDecompressTask(c *gin.Context) seria } // 创建任务 - job, err := task.NewDecompressTask(fs.User, service.Src, service.Dst) + job, err := task.NewDecompressTask(fs.User, service.Src, service.Dst, service.Encoding) if err != nil { return serializer.Err(serializer.CodeNotSet, "任务创建失败", err) }