mirror of
https://github.com/project-zot/zot.git
synced 2024-12-16 21:56:37 -05:00
Merge pull request #112 from rchincha/gc
gc: add a policy to skip garbage collecting new blobs
This commit is contained in:
commit
811a424858
6 changed files with 50 additions and 6 deletions
|
@ -1504,8 +1504,8 @@ go_repository(
|
|||
go_repository(
|
||||
name = "com_github_opencontainers_umoci",
|
||||
importpath = "github.com/opencontainers/umoci",
|
||||
sum = "h1:nUULYM+jSLLJCVN2gd4wldm8/yuVMahC36UXna3jEqI=",
|
||||
version = "v0.4.6",
|
||||
sum = "h1:6JA6emg6B0/8EhBw8i010nkXCnB1skxW+FQPFDJDzoA=",
|
||||
version = "v0.4.7-0.20200704224433-977db481b72c",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
|
|
3
go.mod
3
go.mod
|
@ -4,6 +4,7 @@ go 1.14
|
|||
|
||||
require (
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
|
||||
github.com/apex/log v1.4.0
|
||||
github.com/briandowns/spinner v1.11.1
|
||||
github.com/chartmuseum/auth v0.4.0
|
||||
github.com/dustin/go-humanize v1.0.0
|
||||
|
@ -21,7 +22,7 @@ require (
|
|||
github.com/opencontainers/distribution-spec v1.0.0-rc0
|
||||
github.com/opencontainers/go-digest v1.0.0
|
||||
github.com/opencontainers/image-spec v1.0.1
|
||||
github.com/opencontainers/umoci v0.4.6
|
||||
github.com/opencontainers/umoci v0.4.7-0.20200704224433-977db481b72c
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
|
||||
github.com/rs/zerolog v1.17.2
|
||||
github.com/smartystreets/goconvey v1.6.4
|
||||
|
|
4
go.sum
4
go.sum
|
@ -151,6 +151,8 @@ github.com/klauspost/compress v1.10.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
|
|||
github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
|
||||
github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
|
@ -217,6 +219,8 @@ github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNia
|
|||
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
|
||||
github.com/opencontainers/umoci v0.4.6 h1:nUULYM+jSLLJCVN2gd4wldm8/yuVMahC36UXna3jEqI=
|
||||
github.com/opencontainers/umoci v0.4.6/go.mod h1:ZyTwgJPvYl9xv1Cf3ykU41BxMSCxW3AtoueA5Bmxs1E=
|
||||
github.com/opencontainers/umoci v0.4.7-0.20200704224433-977db481b72c h1:6JA6emg6B0/8EhBw8i010nkXCnB1skxW+FQPFDJDzoA=
|
||||
github.com/opencontainers/umoci v0.4.7-0.20200704224433-977db481b72c/go.mod h1:ZyTwgJPvYl9xv1Cf3ykU41BxMSCxW3AtoueA5Bmxs1E=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
|
||||
|
|
|
@ -33,7 +33,7 @@ go_library(
|
|||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
timeout = "short",
|
||||
timeout = "moderate",
|
||||
srcs = ["controller_test.go"],
|
||||
data = [
|
||||
"//:exported_testdata",
|
||||
|
|
|
@ -11,10 +11,12 @@ go_library(
|
|||
deps = [
|
||||
"//errors:go_default_library",
|
||||
"//pkg/log:go_default_library",
|
||||
"@com_github_apex_log//:go_default_library",
|
||||
"@com_github_gofrs_uuid//:go_default_library",
|
||||
"@com_github_opencontainers_go_digest//:go_default_library",
|
||||
"@com_github_opencontainers_image_spec//specs-go/v1:go_default_library",
|
||||
"@com_github_opencontainers_umoci//:go_default_library",
|
||||
"@com_github_opencontainers_umoci//oci/casext:go_default_library",
|
||||
"@com_github_rs_zerolog//:go_default_library",
|
||||
"@io_etcd_go_bbolt//:go_default_library",
|
||||
],
|
||||
|
|
|
@ -11,13 +11,16 @@ import (
|
|||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/anuvu/zot/errors"
|
||||
zlog "github.com/anuvu/zot/pkg/log"
|
||||
apexlog "github.com/apex/log"
|
||||
guuid "github.com/gofrs/uuid"
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/opencontainers/umoci"
|
||||
"github.com/opencontainers/umoci/oci/casext"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
|
@ -25,6 +28,7 @@ const (
|
|||
// BlobUploadDir defines the upload directory for blob uploads.
|
||||
BlobUploadDir = ".uploads"
|
||||
schemaVersion = 2
|
||||
gcDelay = 1 * time.Hour
|
||||
)
|
||||
|
||||
// BlobUpload models and upload request.
|
||||
|
@ -66,6 +70,20 @@ func NewImageStore(rootDir string, gc bool, dedupe bool, log zlog.Logger) *Image
|
|||
is.cache = NewCache(rootDir, "cache", log)
|
||||
}
|
||||
|
||||
if gc {
|
||||
// we use umoci GC to perform garbage-collection, but it uses its own logger
|
||||
// - so capture those logs, could be useful
|
||||
apexlog.SetLevel(apexlog.DebugLevel)
|
||||
apexlog.SetHandler(apexlog.HandlerFunc(func(entry *apexlog.Entry) error {
|
||||
e := log.Debug()
|
||||
for k, v := range entry.Fields {
|
||||
e = e.Interface(k, v)
|
||||
}
|
||||
e.Msg(entry.Message)
|
||||
return nil
|
||||
}))
|
||||
}
|
||||
|
||||
return is
|
||||
}
|
||||
|
||||
|
@ -492,7 +510,7 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType
|
|||
}
|
||||
defer oci.Close()
|
||||
|
||||
if err := oci.GC(context.Background()); err != nil {
|
||||
if err := oci.GC(context.Background(), ifOlderThan(is, repo, gcDelay)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
@ -568,7 +586,7 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error {
|
|||
}
|
||||
defer oci.Close()
|
||||
|
||||
if err := oci.GC(context.Background()); err != nil {
|
||||
if err := oci.GC(context.Background(), ifOlderThan(is, repo, gcDelay)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
@ -1025,3 +1043,22 @@ func ensureDir(dir string, log zerolog.Logger) {
|
|||
log.Panic().Err(err).Str("dir", dir).Msg("unable to create dir")
|
||||
}
|
||||
}
|
||||
|
||||
func ifOlderThan(is *ImageStore, repo string, delay time.Duration) casext.GCPolicy {
|
||||
return func(ctx context.Context, digest godigest.Digest) (bool, error) {
|
||||
blobPath := is.BlobPath(repo, digest)
|
||||
fi, err := os.Stat(blobPath)
|
||||
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if fi.ModTime().Add(delay).After(time.Now()) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
is.log.Info().Str("digest", digest.String()).Str("blobPath", blobPath).Msg("perform GC on blob")
|
||||
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue