gc: add a policy to skip garbage collecting new blobs

We perform inline garbage collection of orphan blobs. However, the
dist-spec poses a problem because blobs begin their life as orphan blobs
and then a manifest is add which refers to these blobs.

We use umoci's GC() to perform garbage collection and policy support
has been added recently which can control whether a blob can be skipped
for GC.

In this patch, we use a time-based policy to skip blobs.

WORKSPACE

@@ -1504,8 +1504,8 @@ go_repository(
 go_repository(
     name = "com_github_opencontainers_umoci",
     importpath = "github.com/opencontainers/umoci",
-    sum = "h1:nUULYM+jSLLJCVN2gd4wldm8/yuVMahC36UXna3jEqI=",
+    sum = "h1:6JA6emg6B0/8EhBw8i010nkXCnB1skxW+FQPFDJDzoA=",
-    version = "v0.4.6",
+    version = "v0.4.7-0.20200704224433-977db481b72c",
 )
 
 go_repository(

go.mod

@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
+	github.com/apex/log v1.4.0
 	github.com/briandowns/spinner v1.11.1
 	github.com/chartmuseum/auth v0.4.0
 	github.com/dustin/go-humanize v1.0.0
@@ -21,7 +22,7 @@ require (
 	github.com/opencontainers/distribution-spec v1.0.0-rc0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1
-	github.com/opencontainers/umoci v0.4.6
+	github.com/opencontainers/umoci v0.4.7-0.20200704224433-977db481b72c
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/rs/zerolog v1.17.2
 	github.com/smartystreets/goconvey v1.6.4

go.sum

@@ -151,6 +151,8 @@ github.com/klauspost/compress v1.10.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
 github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
 github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -217,6 +219,8 @@ github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNia
 github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/umoci v0.4.6 h1:nUULYM+jSLLJCVN2gd4wldm8/yuVMahC36UXna3jEqI=
 github.com/opencontainers/umoci v0.4.6/go.mod h1:ZyTwgJPvYl9xv1Cf3ykU41BxMSCxW3AtoueA5Bmxs1E=
+github.com/opencontainers/umoci v0.4.7-0.20200704224433-977db481b72c h1:6JA6emg6B0/8EhBw8i010nkXCnB1skxW+FQPFDJDzoA=
+github.com/opencontainers/umoci v0.4.7-0.20200704224433-977db481b72c/go.mod h1:ZyTwgJPvYl9xv1Cf3ykU41BxMSCxW3AtoueA5Bmxs1E=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=

pkg/api/BUILD.bazel

@@ -33,7 +33,7 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    timeout = "short",
+    timeout = "moderate",
     srcs = ["controller_test.go"],
     data = [
         "//:exported_testdata",

pkg/storage/BUILD.bazel

@@ -11,10 +11,12 @@ go_library(
     deps = [
         "//errors:go_default_library",
         "//pkg/log:go_default_library",
+        "@com_github_apex_log//:go_default_library",
         "@com_github_gofrs_uuid//:go_default_library",
         "@com_github_opencontainers_go_digest//:go_default_library",
         "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library",
         "@com_github_opencontainers_umoci//:go_default_library",
+        "@com_github_opencontainers_umoci//oci/casext:go_default_library",
         "@com_github_rs_zerolog//:go_default_library",
         "@io_etcd_go_bbolt//:go_default_library",
     ],

pkg/storage/storage.go

@@ -11,13 +11,16 @@ import (
 	"path"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"github.com/anuvu/zot/errors"
 	zlog "github.com/anuvu/zot/pkg/log"
+	apexlog "github.com/apex/log"
 	guuid "github.com/gofrs/uuid"
 	godigest "github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/opencontainers/umoci"
+	"github.com/opencontainers/umoci/oci/casext"
 	"github.com/rs/zerolog"
 )
 
@@ -25,6 +28,7 @@ const (
 	// BlobUploadDir defines the upload directory for blob uploads.
 	BlobUploadDir = ".uploads"
 	schemaVersion = 2
+	gcDelay       = 1 * time.Hour
 )
 
 // BlobUpload models and upload request.
@@ -66,6 +70,20 @@ func NewImageStore(rootDir string, gc bool, dedupe bool, log zlog.Logger) *Image
 		is.cache = NewCache(rootDir, "cache", log)
 	}
 
+	if gc {
+		// we use umoci GC to perform garbage-collection, but it uses its own logger
+		// - so capture those logs, could be useful
+		apexlog.SetLevel(apexlog.DebugLevel)
+		apexlog.SetHandler(apexlog.HandlerFunc(func(entry *apexlog.Entry) error {
+			e := log.Debug()
+			for k, v := range entry.Fields {
+				e = e.Interface(k, v)
+			}
+			e.Msg(entry.Message)
+			return nil
+		}))
+	}
+
 	return is
 }
 
@@ -492,7 +510,7 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType
 		}
 		defer oci.Close()
 
-		if err := oci.GC(context.Background()); err != nil {
+		if err := oci.GC(context.Background(), ifOlderThan(is, repo, gcDelay)); err != nil {
 			return "", err
 		}
 	}
@@ -568,7 +586,7 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error {
 		}
 		defer oci.Close()
 
-		if err := oci.GC(context.Background()); err != nil {
+		if err := oci.GC(context.Background(), ifOlderThan(is, repo, gcDelay)); err != nil {
 			return err
 		}
 	}
@@ -1025,3 +1043,22 @@ func ensureDir(dir string, log zerolog.Logger) {
 		log.Panic().Err(err).Str("dir", dir).Msg("unable to create dir")
 	}
 }
+
+func ifOlderThan(is *ImageStore, repo string, delay time.Duration) casext.GCPolicy {
+	return func(ctx context.Context, digest godigest.Digest) (bool, error) {
+		blobPath := is.BlobPath(repo, digest)
+		fi, err := os.Stat(blobPath)
+
+		if err != nil {
+			return false, err
+		}
+
+		if fi.ModTime().Add(delay).After(time.Now()) {
+			return false, nil
+		}
+
+		is.log.Info().Str("digest", digest.String()).Str("blobPath", blobPath).Msg("perform GC on blob")
+
+		return true, nil
+	}
+}