Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-03-11 02:17:43 -05:00
Code Issues Activity

fix(gc): gc removes unknown manifests (#1762)

Browse source 
without removing its index.json reference

fix that by also reporting if manifests with unknown mediatypes
are referenced in index.json

this will make gc delete manifest blobs with deleteImageManifest() method
instead of deleteBlob(), which also removes index.json entries.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
peusebiu 2023-09-05 19:42:12 +03:00 • committed by GitHub
parent 8e36bfd4d1
commit a0290b4b37
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 188 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
pkg/storage/common/common.go
View file

@ -520,6 +520,12 @@ func IsBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string,
found, _ = IsBlobReferencedInImageIndex(imgStore, repo, digest, indexImage, log)
case ispec.MediaTypeImageManifest:
found, _ = isBlobReferencedInImageManifest(imgStore, repo, digest, desc.Digest, log)
default:
log.Warn().Str("mediatype", desc.MediaType).Msg("unknown media-type")
// should return true for digests found in index.json even if we don't know it's mediatype
if digest == desc.Digest {
found = true
}
}
if found {

173
pkg/storage/local/local_test.go
View file

@ -2157,6 +2157,179 @@ func TestGarbageCollectForImageStore(t *testing.T) {
})
}
func TestGarbageCollectImageUnknownManifest(t *testing.T) {
Convey("Garbage collect with short delay", t, func() {
dir := t.TempDir()
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, 1*time.Second,
storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
storeController := storage.StoreController{
DefaultStore: imgStore,
}
unsupportedMediaType := "application/vnd.oci.artifact.manifest.v1+json"
img := test.CreateRandomImage()
err := test.WriteImageToFileSystem(img, repoName, "v1", storeController)
So(err, ShouldBeNil)
// add image with unsupported media type
artifact := test.CreateRandomImage()
err = test.WriteImageToFileSystem(artifact, repoName, "artifact", storeController)
So(err, ShouldBeNil)
// add referrer with unsupported media type
subjectDesc := img.Descriptor()
referrer := test.CreateRandomImageWith().Subject(&subjectDesc).Build()
err = test.WriteImageToFileSystem(referrer, repoName, referrer.Digest().String(), storeController)
So(err, ShouldBeNil)
// modify artifact media type
artifactBuf, err := os.ReadFile(imgStore.BlobPath(repoName, artifact.Digest()))
So(err, ShouldBeNil)
var artifactManifest ispec.Manifest
err = json.Unmarshal(artifactBuf, &artifactManifest)
So(err, ShouldBeNil)
artifactManifest.MediaType = unsupportedMediaType
artifactBuf, err = json.Marshal(artifactManifest)
So(err, ShouldBeNil)
artifactDigest := godigest.FromBytes(artifactBuf)
err = os.WriteFile(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", artifactDigest.Encoded()),
artifactBuf, storageConstants.DefaultFilePerms)
So(err, ShouldBeNil)
// modify referrer media type
referrerBuf, err := os.ReadFile(imgStore.BlobPath(repoName, referrer.Digest()))
So(err, ShouldBeNil)
var referrerManifest ispec.Manifest
err = json.Unmarshal(referrerBuf, &referrerManifest)
So(err, ShouldBeNil)
referrerManifest.MediaType = unsupportedMediaType
referrerBuf, err = json.Marshal(referrerManifest)
So(err, ShouldBeNil)
referrerDigest := godigest.FromBytes(referrerBuf)
err = os.WriteFile(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", referrerDigest.Encoded()),
referrerBuf, storageConstants.DefaultFilePerms)
So(err, ShouldBeNil)
indexJSONBuf, err := os.ReadFile(path.Join(imgStore.RootDir(), repoName, "index.json"))
So(err, ShouldBeNil)
var indexJSON ispec.Index
err = json.Unmarshal(indexJSONBuf, &indexJSON)
So(err, ShouldBeNil)
for idx, desc := range indexJSON.Manifests {
if desc.Digest == artifact.Digest() {
indexJSON.Manifests[idx].Digest = artifactDigest
indexJSON.Manifests[idx].MediaType = unsupportedMediaType
} else if desc.Digest == referrer.Digest() {
indexJSON.Manifests[idx].Digest = referrerDigest
indexJSON.Manifests[idx].MediaType = unsupportedMediaType
}
}
indexJSONBuf, err = json.Marshal(indexJSON)
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(imgStore.RootDir(), repoName, "index.json"),
indexJSONBuf, storageConstants.DefaultFilePerms)
So(err, ShouldBeNil)
// sleep so orphan blob can be GC'ed
time.Sleep(1 * time.Second)
Convey("Garbage collect blobs referenced by manifest with unsupported media type", func() {
err = imgStore.RunGCRepo(repoName)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest(repoName, img.DigestStr())
So(err, ShouldBeNil)
hasBlob, _, err := imgStore.CheckBlob(repoName, img.ConfigDescriptor.Digest)
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
_, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String())
So(err, ShouldNotBeNil)
_, _, _, err = imgStore.GetImageManifest(repoName, referrerDigest.String())
So(err, ShouldNotBeNil)
hasBlob, _, err = imgStore.CheckBlob(repoName, artifactDigest)
So(err, ShouldNotBeNil)
So(hasBlob, ShouldEqual, false)
hasBlob, _, err = imgStore.CheckBlob(repoName, referrerDigest)
So(err, ShouldNotBeNil)
So(hasBlob, ShouldEqual, false)
hasBlob, _, err = imgStore.CheckBlob(repoName, artifact.ConfigDescriptor.Digest)
So(err, ShouldNotBeNil)
So(hasBlob, ShouldEqual, false)
hasBlob, _, err = imgStore.CheckBlob(repoName, referrer.ConfigDescriptor.Digest)
So(err, ShouldNotBeNil)
So(hasBlob, ShouldEqual, false)
})
Convey("Garbage collect - gc repo after manifest delete", func() {
err = imgStore.DeleteImageManifest(repoName, img.DigestStr(), true)
So(err, ShouldBeNil)
err = imgStore.RunGCRepo(repoName)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest(repoName, img.DigestStr())
So(err, ShouldNotBeNil)
hasBlob, _, err := imgStore.CheckBlob(repoName, img.ConfigDescriptor.Digest)
So(err, ShouldNotBeNil)
So(hasBlob, ShouldEqual, false)
_, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String())
So(err, ShouldNotBeNil)
_, _, _, err = imgStore.GetImageManifest(repoName, referrerDigest.String())
So(err, ShouldNotBeNil)
hasBlob, _, err = imgStore.CheckBlob(repoName, artifactDigest)
So(err, ShouldNotBeNil)
So(hasBlob, ShouldEqual, false)
hasBlob, _, err = imgStore.CheckBlob(repoName, referrerDigest)
So(err, ShouldNotBeNil)
So(hasBlob, ShouldEqual, false)
hasBlob, _, err = imgStore.CheckBlob(repoName, artifact.ConfigDescriptor.Digest)
So(err, ShouldNotBeNil)
So(hasBlob, ShouldEqual, false)
hasBlob, _, err = imgStore.CheckBlob(repoName, referrer.ConfigDescriptor.Digest)
So(err, ShouldNotBeNil)
So(hasBlob, ShouldEqual, false)
})
})
}
func TestGarbageCollectErrors(t *testing.T) {
Convey("Make image store", t, func(c C) {
dir := t.TempDir()

18
pkg/storage/storage_test.go
View file

@ -1485,7 +1485,7 @@ func TestGarbageCollectImageManifest(t *testing.T) {
So(err, ShouldBeNil)
// sleep so orphan blob can be GC'ed
time.Sleep(5 * time.Second)
time.Sleep(1 * time.Second)
// upload blob
upload, err = imgStore.NewBlobUpload(repoName)
@ -1657,7 +1657,7 @@ func TestGarbageCollectImageManifest(t *testing.T) {
So(hasBlob, ShouldEqual, true)
// sleep so orphan blob can be GC'ed
time.Sleep(5 * time.Second)
time.Sleep(1 * time.Second)
Convey("Garbage collect blobs after manifest is removed", func() {
err = imgStore.DeleteImageManifest(repoName, digest.String(), false)
@ -1735,7 +1735,7 @@ func TestGarbageCollectImageManifest(t *testing.T) {
// garbage-collect is repo-local and dedupe is global and they can interact in strange ways
var imgStore storageTypes.ImageStore
gcDelay := 5 * time.Second
gcDelay := 3 * time.Second
if testcase.storageType == storageConstants.S3StorageDriverName {
skipIt(t)
@ -1823,13 +1823,13 @@ func TestGarbageCollectImageManifest(t *testing.T) {
_, _, err = imgStore.PutImageManifest(repo1Name, tag, ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
// sleep so past GC timeout
time.Sleep(3 * time.Second)
hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest)
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
// sleep so past GC timeout
time.Sleep(10 * time.Second)
hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest)
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
@ -2092,8 +2092,8 @@ func TestGarbageCollectImageIndex(t *testing.T) {
Convey("Garbage collect with short delay", func() {
var imgStore storageTypes.ImageStore
gcDelay := 5 * time.Second
imageRetentionDelay := 5 * time.Second
gcDelay := 2 * time.Second
imageRetentionDelay := 2 * time.Second
if testcase.storageType == storageConstants.S3StorageDriverName {
skipIt(t)
@ -2278,7 +2278,7 @@ func TestGarbageCollectImageIndex(t *testing.T) {
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
time.Sleep(5 * time.Second)
time.Sleep(2 * time.Second)
Convey("delete inner referenced manifest", func() {
err = imgStore.RunGCRepo(repoName)