Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

gc: add a policy to skip garbage collecting new blobs

Browse source 
We perform inline garbage collection of orphan blobs. However, the
dist-spec poses a problem because blobs begin their life as orphan blobs
and then a manifest is add which refers to these blobs.

We use umoci's GC() to perform garbage collection and policy support
has been added recently which can control whether a blob can be skipped
for GC.

In this patch, we use a time-based policy to skip blobs.
This commit is contained in:
Ramkumar Chinchani 2020-06-30 10:56:58 -07:00
parent 80244f1282
commit 324a517ea3
6 changed files with 50 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
WORKSPACE
View file

@ -1504,8 +1504,8 @@ go_repository(
go_repository(
name = "com_github_opencontainers_umoci",
importpath = "github.com/opencontainers/umoci",
sum = "h1:nUULYM+jSLLJCVN2gd4wldm8/yuVMahC36UXna3jEqI=",
version = "v0.4.6",
sum = "h1:6JA6emg6B0/8EhBw8i010nkXCnB1skxW+FQPFDJDzoA=",
version = "v0.4.7-0.20200704224433-977db481b72c",
)
go_repository(

3
go.mod
View file

@ -4,6 +4,7 @@ go 1.14
require (
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
github.com/apex/log v1.4.0
github.com/briandowns/spinner v1.11.1
github.com/chartmuseum/auth v0.4.0
github.com/dustin/go-humanize v1.0.0
@ -21,7 +22,7 @@ require (
github.com/opencontainers/distribution-spec v1.0.0-rc0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.1
github.com/opencontainers/umoci v0.4.6
github.com/opencontainers/umoci v0.4.7-0.20200704224433-977db481b72c
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
github.com/rs/zerolog v1.17.2
github.com/smartystreets/goconvey v1.6.4

4
go.sum
View file

@ -151,6 +151,8 @@ github.com/klauspost/compress v1.10.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@ -217,6 +219,8 @@ github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNia
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/opencontainers/umoci v0.4.6 h1:nUULYM+jSLLJCVN2gd4wldm8/yuVMahC36UXna3jEqI=
github.com/opencontainers/umoci v0.4.6/go.mod h1:ZyTwgJPvYl9xv1Cf3ykU41BxMSCxW3AtoueA5Bmxs1E=
github.com/opencontainers/umoci v0.4.7-0.20200704224433-977db481b72c h1:6JA6emg6B0/8EhBw8i010nkXCnB1skxW+FQPFDJDzoA=
github.com/opencontainers/umoci v0.4.7-0.20200704224433-977db481b72c/go.mod h1:ZyTwgJPvYl9xv1Cf3ykU41BxMSCxW3AtoueA5Bmxs1E=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=

2
pkg/api/BUILD.bazel
View file

@ -33,7 +33,7 @@ go_library(
go_test(
name = "go_default_test",
timeout = "short",
timeout = "moderate",
srcs = ["controller_test.go"],
data = [
"//:exported_testdata",

2
pkg/storage/BUILD.bazel
View file

@ -11,10 +11,12 @@ go_library(
deps = [
"//errors:go_default_library",
"//pkg/log:go_default_library",
"@com_github_apex_log//:go_default_library",
"@com_github_gofrs_uuid//:go_default_library",
"@com_github_opencontainers_go_digest//:go_default_library",
"@com_github_opencontainers_image_spec//specs-go/v1:go_default_library",
"@com_github_opencontainers_umoci//:go_default_library",
"@com_github_opencontainers_umoci//oci/casext:go_default_library",
"@com_github_rs_zerolog//:go_default_library",
"@io_etcd_go_bbolt//:go_default_library",
],

41
pkg/storage/storage.go
View file

@ -11,13 +11,16 @@ import (
"path"
"path/filepath"
"sync"
"time"
"github.com/anuvu/zot/errors"
zlog "github.com/anuvu/zot/pkg/log"
apexlog "github.com/apex/log"
guuid "github.com/gofrs/uuid"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/opencontainers/umoci"
"github.com/opencontainers/umoci/oci/casext"
"github.com/rs/zerolog"
)
@ -25,6 +28,7 @@ const (
// BlobUploadDir defines the upload directory for blob uploads.
BlobUploadDir = ".uploads"
schemaVersion = 2
gcDelay = 1 * time.Hour
)
// BlobUpload models and upload request.
@ -66,6 +70,20 @@ func NewImageStore(rootDir string, gc bool, dedupe bool, log zlog.Logger) *Image
is.cache = NewCache(rootDir, "cache", log)
}
if gc {
// we use umoci GC to perform garbage-collection, but it uses its own logger
// - so capture those logs, could be useful
apexlog.SetLevel(apexlog.DebugLevel)
apexlog.SetHandler(apexlog.HandlerFunc(func(entry *apexlog.Entry) error {
e := log.Debug()
for k, v := range entry.Fields {
e = e.Interface(k, v)
}
e.Msg(entry.Message)
return nil
}))
}
return is
}
@ -492,7 +510,7 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType
}
defer oci.Close()
if err := oci.GC(context.Background()); err != nil {
if err := oci.GC(context.Background(), ifOlderThan(is, repo, gcDelay)); err != nil {
return "", err
}
}
@ -568,7 +586,7 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error {
}
defer oci.Close()
if err := oci.GC(context.Background()); err != nil {
if err := oci.GC(context.Background(), ifOlderThan(is, repo, gcDelay)); err != nil {
return err
}
}
@ -1025,3 +1043,22 @@ func ensureDir(dir string, log zerolog.Logger) {
log.Panic().Err(err).Str("dir", dir).Msg("unable to create dir")
}
}
func ifOlderThan(is *ImageStore, repo string, delay time.Duration) casext.GCPolicy {
return func(ctx context.Context, digest godigest.Digest) (bool, error) {
blobPath := is.BlobPath(repo, digest)
fi, err := os.Stat(blobPath)
if err != nil {
return false, err
}
if fi.ModTime().Add(delay).After(time.Now()) {
return false, nil
}
is.log.Info().Str("digest", digest.String()).Str("blobPath", blobPath).Msg("perform GC on blob")
return true, nil
}
}