0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-30 22:34:13 -05:00
zot/pkg/storage/gc/gc_internal_test.go
Andrei Aaron ba6f347d8d
refactor(pkg/test): split logic in pkg/test/common.go into multiple packages (#1861)
Which could be imported independently. See more details:
1. "zotregistry.io/zot/pkg/test/common" - currently used as
   tcommon "zotregistry.io/zot/pkg/test/common" - inside pkg/test
   test "zotregistry.io/zot/pkg/test/common" - in tests
   . "zotregistry.io/zot/pkg/test/common" - in tests
Decouple zb from code in test/pkg in order to keep the size small.

2. "zotregistry.io/zot/pkg/test/image-utils" - curently used as
   . "zotregistry.io/zot/pkg/test/image-utils"

3. "zotregistry.io/zot/pkg/test/deprecated" -  curently used as
   "zotregistry.io/zot/pkg/test/deprecated"
This one will bre replaced gradually by image-utils in the future.

4. "zotregistry.io/zot/pkg/test/signature" - (cosign + notation) use as
   "zotregistry.io/zot/pkg/test/signature"

5. "zotregistry.io/zot/pkg/test/auth" - (bearer + oidc)  curently used as
   authutils "zotregistry.io/zot/pkg/test/auth"

 6. "zotregistry.io/zot/pkg/test/oci-utils" -  curently used as
   ociutils "zotregistry.io/zot/pkg/test/oci-utils"

Some unused functions were removed, some were replaced, and in
a few cases specific funtions were moved to the files they were used in.

Added an interface for the StoreController, this reduces the number of imports
of the entire image store, decreasing binary size for tests.
If the zb code was still coupled with pkg/test, this would have reflected in zb size.

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
2023-09-27 11:34:48 -07:00

567 lines
16 KiB
Go

package gc
import (
"bytes"
"encoding/json"
"errors"
"os"
"path"
"testing"
"time"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/rs/zerolog"
. "github.com/smartystreets/goconvey/convey"
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
common "zotregistry.io/zot/pkg/storage/common"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
. "zotregistry.io/zot/pkg/test/image-utils"
"zotregistry.io/zot/pkg/test/mocks"
)
var (
errGC = errors.New("gc error")
repoName = "test" //nolint: gochecknoglobals
)
func TestGarbageCollectManifestErrors(t *testing.T) {
Convey("Make imagestore and upload manifest", t, func(c C) {
dir := t.TempDir()
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver)
gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{
Referrers: true,
Delay: storageConstants.DefaultGCDelay,
RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
}, log)
Convey("trigger repo not found in addImageIndexBlobsToReferences()", func() {
err := gc.addIndexBlobsToReferences(repoName, ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: godigest.FromString("miss"),
MediaType: ispec.MediaTypeImageIndex,
},
},
}, map[string]bool{})
So(err, ShouldNotBeNil)
})
Convey("trigger repo not found in addImageManifestBlobsToReferences()", func() {
err := gc.addIndexBlobsToReferences(repoName, ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: godigest.FromString("miss"),
MediaType: ispec.MediaTypeImageManifest,
},
},
}, map[string]bool{})
So(err, ShouldNotBeNil)
})
Convey("trigger repo not found in addORASImageManifestBlobsToReferences()", func() {
err := gc.addIndexBlobsToReferences(repoName, ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: godigest.FromString("miss"),
MediaType: artifactspec.MediaTypeArtifactManifest,
},
},
}, map[string]bool{})
So(err, ShouldNotBeNil)
})
content := []byte("this is a blob")
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
_, blen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), digest)
So(err, ShouldBeNil)
So(blen, ShouldEqual, len(content))
cblob, cdigest := GetRandomImageConfig()
_, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest)
So(err, ShouldBeNil)
So(clen, ShouldEqual, len(cblob))
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeImageConfig,
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageLayer,
Digest: digest,
Size: int64(len(content)),
},
},
}
manifest.SchemaVersion = 2
body, err := json.Marshal(manifest)
So(err, ShouldBeNil)
manifestDigest := godigest.FromBytes(body)
_, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, body)
So(err, ShouldBeNil)
Convey("trigger GetIndex error in GetReferencedBlobs", func() {
index, err := common.GetIndex(imgStore, repoName, log)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o000)
So(err, ShouldBeNil)
defer func() {
err := os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o755)
So(err, ShouldBeNil)
}()
err = gc.addIndexBlobsToReferences(repoName, index, map[string]bool{})
So(err, ShouldNotBeNil)
})
Convey("trigger GetImageManifest error in AddIndexBlobsToReferences", func() {
index, err := common.GetIndex(imgStore, repoName, log)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o000)
So(err, ShouldBeNil)
defer func() {
err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o755)
So(err, ShouldBeNil)
}()
err = gc.addIndexBlobsToReferences(repoName, index, map[string]bool{})
So(err, ShouldNotBeNil)
})
})
}
func TestGarbageCollectIndexErrors(t *testing.T) {
Convey("Make imagestore and upload manifest", t, func(c C) {
dir := t.TempDir()
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver)
gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{
Referrers: true,
Delay: storageConstants.DefaultGCDelay,
RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
}, log)
content := []byte("this is a blob")
bdgst := godigest.FromBytes(content)
So(bdgst, ShouldNotBeNil)
_, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst)
So(err, ShouldBeNil)
So(bsize, ShouldEqual, len(content))
var index ispec.Index
index.SchemaVersion = 2
index.MediaType = ispec.MediaTypeImageIndex
var digest godigest.Digest
for i := 0; i < 4; i++ {
// upload image config blob
upload, err := imgStore.NewBlobUpload(repoName)
So(err, ShouldBeNil)
So(upload, ShouldNotBeEmpty)
cblob, cdigest := GetRandomImageConfig()
buf := bytes.NewBuffer(cblob)
buflen := buf.Len()
blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
// create a manifest
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeImageConfig,
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageLayer,
Digest: bdgst,
Size: bsize,
},
},
}
manifest.SchemaVersion = 2
content, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
_, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content)
So(err, ShouldBeNil)
index.Manifests = append(index.Manifests, ispec.Descriptor{
Digest: digest,
MediaType: ispec.MediaTypeImageManifest,
Size: int64(len(content)),
})
}
// upload index image
indexContent, err := json.Marshal(index)
So(err, ShouldBeNil)
indexDigest := godigest.FromBytes(indexContent)
So(indexDigest, ShouldNotBeNil)
_, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent)
So(err, ShouldBeNil)
index, err = common.GetIndex(imgStore, repoName, log)
So(err, ShouldBeNil)
err = gc.addIndexBlobsToReferences(repoName, index, map[string]bool{})
So(err, ShouldBeNil)
Convey("trigger GetImageIndex error in GetReferencedBlobsInImageIndex", func() {
err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o000)
So(err, ShouldBeNil)
defer func() {
err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o755)
So(err, ShouldBeNil)
}()
err = gc.addIndexBlobsToReferences(repoName, index, map[string]bool{})
So(err, ShouldNotBeNil)
})
})
}
func TestGarbageCollectWithMockedImageStore(t *testing.T) {
Convey("Cover gc error paths", t, func(c C) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
Convey("Error on PutIndexContent in gc.cleanRepo()", func() {
returnedIndexJSON := ispec.Index{}
returnedIndexJSONBuf, err := json.Marshal(returnedIndexJSON)
So(err, ShouldBeNil)
imgStore := mocks.MockedImageStore{
PutIndexContentFn: func(repo string, index ispec.Index) error {
return errGC
},
GetIndexContentFn: func(repo string) ([]byte, error) {
return returnedIndexJSONBuf, nil
},
}
gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{
Referrers: true,
Delay: storageConstants.DefaultGCDelay,
RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
}, log)
err = gc.cleanRepo(repoName)
So(err, ShouldNotBeNil)
})
Convey("Error on gc.cleanBlobs() in gc.cleanRepo()", func() {
returnedIndexJSON := ispec.Index{}
returnedIndexJSONBuf, err := json.Marshal(returnedIndexJSON)
So(err, ShouldBeNil)
imgStore := mocks.MockedImageStore{
PutIndexContentFn: func(repo string, index ispec.Index) error {
return nil
},
GetIndexContentFn: func(repo string) ([]byte, error) {
return returnedIndexJSONBuf, nil
},
GetAllBlobsFn: func(repo string) ([]string, error) {
return []string{}, errGC
},
}
gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{
Referrers: true,
Delay: storageConstants.DefaultGCDelay,
RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
}, log)
err = gc.cleanRepo(repoName)
So(err, ShouldNotBeNil)
})
Convey("False on imgStore.DirExists() in gc.cleanRepo()", func() {
imgStore := mocks.MockedImageStore{
DirExistsFn: func(d string) bool {
return false
},
}
gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{
Referrers: true,
Delay: storageConstants.DefaultGCDelay,
RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
}, log)
err := gc.cleanRepo(repoName)
So(err, ShouldNotBeNil)
})
Convey("Error on gc.identifyManifestsReferencedInIndex in gc.cleanManifests() with multiarch image", func() {
indexImageDigest := godigest.FromBytes([]byte("digest"))
returnedIndexImage := ispec.Index{
Subject: &ispec.DescriptorEmptyJSON,
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageIndex,
Digest: godigest.FromBytes([]byte("digest2")),
},
},
}
returnedIndexImageBuf, err := json.Marshal(returnedIndexImage)
So(err, ShouldBeNil)
imgStore := mocks.MockedImageStore{
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
if digest == indexImageDigest {
return returnedIndexImageBuf, nil
} else {
return nil, errGC
}
},
}
gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{
Referrers: false,
Delay: storageConstants.DefaultGCDelay,
RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
}, log)
err = gc.cleanManifests(repoName, &ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageIndex,
Digest: indexImageDigest,
},
},
})
So(err, ShouldNotBeNil)
})
Convey("Error on gc.identifyManifestsReferencedInIndex in gc.cleanManifests() with image", func() {
imgStore := mocks.MockedImageStore{
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return nil, errGC
},
}
gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{
Referrers: false,
Delay: storageConstants.DefaultGCDelay,
RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
}, log)
err := gc.cleanManifests(repoName, &ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageManifest,
Digest: godigest.FromBytes([]byte("digest")),
},
},
})
So(err, ShouldNotBeNil)
})
Convey("Error on gc.gcManifest() in gc.cleanManifests() with image", func() {
returnedImage := ispec.Manifest{
MediaType: ispec.MediaTypeImageManifest,
}
returnedImageBuf, err := json.Marshal(returnedImage)
So(err, ShouldBeNil)
imgStore := mocks.MockedImageStore{
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return returnedImageBuf, nil
},
}
metaDB := mocks.MetaDBMock{
RemoveRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest) error {
return errGC
},
}
gc := NewGarbageCollect(imgStore, metaDB, Options{
Referrers: false,
Delay: storageConstants.DefaultGCDelay,
RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
}, log)
err = gc.cleanManifests(repoName, &ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageManifest,
Digest: godigest.FromBytes([]byte("digest")),
},
},
})
So(err, ShouldNotBeNil)
})
Convey("Error on gc.gcManifest() in gc.cleanManifests() with signature", func() {
returnedImage := ispec.Manifest{
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: zcommon.NotationSignature,
}
returnedImageBuf, err := json.Marshal(returnedImage)
So(err, ShouldBeNil)
imgStore := mocks.MockedImageStore{
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return returnedImageBuf, nil
},
}
metaDB := mocks.MetaDBMock{
DeleteSignatureFn: func(repo string, signedManifestDigest godigest.Digest, sm types.SignatureMetadata) error {
return errGC
},
}
gc := NewGarbageCollect(imgStore, metaDB, Options{
Referrers: false,
Delay: storageConstants.DefaultGCDelay,
RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
}, log)
desc := ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Digest: godigest.FromBytes([]byte("digest")),
}
index := &ispec.Index{
Manifests: []ispec.Descriptor{desc},
}
_, err = gc.removeManifest(repoName, index, desc, storage.NotationType,
godigest.FromBytes([]byte("digest2")))
So(err, ShouldNotBeNil)
})
Convey("Error on gc.gcReferrer() in gc.cleanManifests() with image index", func() {
manifestDesc := ispec.Descriptor{
MediaType: ispec.MediaTypeImageIndex,
Digest: godigest.FromBytes([]byte("digest")),
}
returnedIndexImage := ispec.Index{
MediaType: ispec.MediaTypeImageIndex,
Subject: &ispec.Descriptor{
Digest: godigest.FromBytes([]byte("digest2")),
},
Manifests: []ispec.Descriptor{
manifestDesc,
},
}
returnedIndexImageBuf, err := json.Marshal(returnedIndexImage)
So(err, ShouldBeNil)
imgStore := mocks.MockedImageStore{
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return returnedIndexImageBuf, nil
},
StatBlobFn: func(repo string, digest godigest.Digest) (bool, int64, time.Time, error) {
return false, -1, time.Time{}, errGC
},
}
gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{
Referrers: true,
Delay: storageConstants.DefaultGCDelay,
RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
}, log)
err = gc.cleanManifests(repoName, &returnedIndexImage)
So(err, ShouldNotBeNil)
})
Convey("Error on gc.gcReferrer() in gc.cleanManifests() with image", func() {
manifestDesc := ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Digest: godigest.FromBytes([]byte("digest")),
}
returnedImage := ispec.Manifest{
Subject: &ispec.Descriptor{
Digest: godigest.FromBytes([]byte("digest2")),
},
MediaType: ispec.MediaTypeImageManifest,
}
returnedImageBuf, err := json.Marshal(returnedImage)
So(err, ShouldBeNil)
imgStore := mocks.MockedImageStore{
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return returnedImageBuf, nil
},
StatBlobFn: func(repo string, digest godigest.Digest) (bool, int64, time.Time, error) {
return false, -1, time.Time{}, errGC
},
}
gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{
Referrers: true,
Delay: storageConstants.DefaultGCDelay,
RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
}, log)
err = gc.cleanManifests(repoName, &ispec.Index{
Manifests: []ispec.Descriptor{
manifestDesc,
},
})
So(err, ShouldNotBeNil)
})
})
}